{"crawledAt":"2026-10-13T12:55:11.870Z","competitions":[{"id":"assetopsbench","name":"AssetOpsBench","tagline":"A unified framework for building, orchestrating, and evaluating domain-specific AI agents in Industry 4.0.","description":"AssetOpsBench is a benchmark for evaluating AI agents on multi-step industrial operations and maintenance tasks. It features domain-specific Model Context Protocol (MCP) servers and evaluates agents across 141+ scenarios. Trajectories are scored using an LLM judge across six dimensions.","url":"https://github.com/IBM/AssetOpsBench","host":"github","status":"active","domain":"llm-eval","tags":["ai-agents","industry-4-0","mcp","time-series","benchmark","multi-agent"],"images":["https://github.com/user-attachments/assets/615059be-e296-40d3-90ec-97ee6cb00412"],"participation":{"howToSubmit":"Submit agent trajectories to active challenges on CodaBench/Kaggle, or contribute new scenarios via GitHub Pull Requests.","deadline":"rolling","cost":"free","requirements":["Python 3.12+","uv","Docker"],"compute":"unknown"},"links":[{"label":"Paper","url":"https://arxiv.org/pdf/2506.03828"},{"label":"Dataset","url":"https://huggingface.co/datasets/ibm-research/AssetOpsBench"},{"label":"Playground","url":"https://huggingface.co/spaces/ibm-research/AssetOps-Bench"},{"label":"Kaggle","url":"https://www.kaggle.com/benchmarks/ibm-research/asset-ops-bench"},{"label":"CODS 2025 Competition","url":"https://www.codabench.org/competitions/10206/"},{"label":"IJCAI 2026 Challenge","url":"https://sites.google.com/view/ai-industrial-challenge-ijcai/home"}],"organizer":{"name":"IBM Research","url":"https://research.ibm.com/","type":"company"},"quickstart":"git clone https://github.com/IBM/AssetOpsBench.git\ncd AssetOpsBench\nuv sync\npython -m assetopsbench.run --scenario \"List all sensors of Chiller 6 in MAIN site\"","problems":[{"description":"Multi-step end-to-end workflows evaluating AI agents on industrial asset operations and maintenance tasks across 9 asset classes and 141+ scenarios.","id":"industrial-agent-scenarios","name":"Industrial Agent Scenarios","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"IBM","name":"AssetOpsBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.132Z"},{"id":"alpaca-eval","name":"AlpacaEval","tagline":"An automatic evaluator for instruction-following language models.","description":"AlpacaEval is an LLM-based automatic evaluation pipeline that is fast, cheap, and highly correlated with human preferences. It measures the win rate of a model's outputs against a reference baseline using an LLM judge. Run by Tatsu Lab, it provides a standardized benchmark to accelerate the development of instruction-following models.","url":"https://github.com/tatsu-lab/alpaca_eval","host":"github","status":"active","domain":"llm-eval","tags":["llm-eval","instruction-following","benchmark","nlp","automated-evaluation"],"images":["https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/chat_correlations_no_ae.png","https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/length_gameability.png"],"participation":{"howToSubmit":"Add a model config definition in the models_configs folder, run evaluation using `alpaca_eval evaluate_from_model`, and open a PR with the config, outputs file, and updated leaderboard.","submissionFormat":"A YAML model config, a JSON file containing model outputs, and updated leaderboard CSV files.","computeDetails":"API-based evaluation using OpenAI/Anthropic keys. Local models can be evaluated using HuggingFace or vLLM.","deadline":"rolling","cost":"free","verification":"Core maintainers verify top-5 models by decoding and evaluating them directly.","requirements":["Python 3.10+","OpenAI API Key"],"compute":"none"},"links":[{"label":"Website","url":"https://tatsu-lab.github.io/alpaca_eval/"},{"label":"Paper (AlpacaEval 2.0)","url":"https://arxiv.org/abs/2404.04475"},{"label":"Paper (AlpacaFarm)","url":"https://arxiv.org/abs/2305.14387"},{"label":"Dataset","url":"https://huggingface.co/datasets/tatsu-lab/alpaca_eval"}],"organizer":{"name":"Tatsu Lab","url":"https://tatsu-lab.github.io/","type":"academic"},"quickstart":"pip install alpaca-eval\nexport OPENAI_API_KEY=<your_api_key>\nalpaca_eval --model_outputs 'example/outputs.json'","problems":[{"description":"Evaluates models using weighted_alpaca_eval_gpt4_turbo as the annotator and gpt4_turbo as the baseline, controlling for length bias.","metricUnit":"%","id":"alpaca-eval-2","name":"AlpacaEval 2.0 (Length-Controlled Win Rate)","metricName":"win_rate","metricDirection":"maximize","baseline":50,"isOpen":true,"records":[],"contested":false},{"description":"Evaluates models using alpaca_eval_gpt4 as the annotator and text_davinci_003 as the baseline.","metricUnit":"%","id":"alpaca-eval-1","name":"AlpacaEval 1.0 (text-davinci-003 baseline)","metricName":"win_rate","metricDirection":"maximize","baseline":50,"isOpen":false,"records":[{"contributor":"gpt4","value":95.3,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"claude","value":88.4,"contributorKind":"model","agent":{"model":"claude","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"chatgpt","value":86.1,"contributorKind":"model","agent":{"model":"chatgpt","family":"openai","confidence":"high","role":"subject"}},{"contributor":"guanaco-65b","value":71.8,"contributorKind":"model","agent":{"model":"guanaco-65b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"vicuna-13b","value":70.4,"contributorKind":"model","agent":{"model":"vicuna-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"text_davinci_003","value":50,"contributorKind":"model","isBaseline":true,"agent":{"model":"text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca-farm-ppo-human","value":41.2,"contributorKind":"model","agent":{"model":"alpaca-farm-ppo-human","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"alpaca-7b","value":26.5,"contributorKind":"model","agent":{"model":"alpaca-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"text_davinci_001","value":15.2,"contributorKind":"model","agent":{"model":"text-davinci-001","family":"openai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Evaluates different automatic annotators on the AlpacaEval set by comparing to 2.5K human annotations.","metricUnit":"%","id":"alpaca-eval-evaluators","name":"Evaluator Meta-Evaluation (Human Agreement)","metricName":"Human agreement","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"alpaca_eval_gpt4","value":69.2,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_cot_gpt4_turbo_fn","value":68.6,"contributorKind":"method","agent":{"model":"gpt-4-turbo","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_gpt4_turbo_fn","value":68.1,"contributorKind":"method","agent":{"model":"gpt-4-turbo","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_llama3_70b_fn","value":67.5,"contributorKind":"method","agent":{"model":"llama-3-70b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt4","value":66.9,"contributorKind":"model","agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_farm_greedy_gpt4","value":66.4,"contributorKind":"method","agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_cot_gpt4_turbo_fn","value":65.7,"contributorKind":"method","agent":{"model":"gpt-4-turbo","family":"openai","confidence":"high","role":"subject"}},{"contributor":"humans","value":65.7,"contributorKind":"unknown","agent":{"evidence":"contributor: humans","family":"human","confidence":"high","role":"subject"}},{"contributor":"claude","value":65.3,"contributorKind":"model","agent":{"model":"claude","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"lmsys_gpt4","value":65.3,"contributorKind":"method","agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"text_davinci_003","value":64.1,"contributorKind":"model","agent":{"model":"text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"longest","value":62.2,"contributorKind":"method","agent":{"evidence":"contributor: longest","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chatgpt","value":57.3,"contributorKind":"model","agent":{"model":"chatgpt","family":"openai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"tatsu-lab","name":"alpaca_eval","branch":"main"},"participants":[{"name":"humans","submissions":1,"problems":["alpaca-eval-evaluators"]}],"stats":{"totalRecords":22,"uniqueParticipants":1,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.132Z"},{"id":"cogdl","name":"CogDL","tagline":"An extensive graph deep learning toolkit to train, evaluate, and benchmark graph neural networks.","description":"CogDL is a graph deep learning toolkit that allows researchers and developers to easily train and compare baseline or customized models for node classification, link prediction, and graph classification. It maintains leaderboards of state-of-the-art results across multiple graph datasets. Submissions are made by implementing models in the toolkit and opening pull requests.","url":"https://github.com/THUDM/CogDL","host":"github","status":"active","domain":"ml-research","tags":["graph-neural-networks","node-classification","link-prediction","graph-classification","benchmark"],"images":[],"participation":{"howToSubmit":"Open an issue and then create a pull request implementing your model in the toolkit and updating the leaderboard in results.md.","submissionFormat":"Model implementation in `cogdl/models`, hyperparameters in `examples/`, and updated results in `results.md`.","computeDetails":"Supports GPU training with custom fast operators like GE-SpMM.","deadline":"rolling","prizes":"recognition only","verification":"CI","requirements":["Python >= 3.7","PyTorch >= 1.7.1"],"compute":"consumer-gpu"},"links":[{"label":"Homepage","url":"https://cogdl.ai/"},{"label":"Paper","url":"https://arxiv.org/abs/2103.00959"},{"label":"Documentation","url":"https://cogdl.readthedocs.io/"},{"label":"Discussion Forum","url":"https://discuss.cogdl.ai/"}],"organizer":{"name":"THUDM","url":"https://cogdl.ai/","type":"academic"},"quickstart":"from cogdl import experiment\n\n# Run GCN on Cora dataset\nexperiment(dataset=\"cora\", model=\"gcn\")","problems":[{"group":"graph-classification","description":"Graph classification on the MUTAG dataset.","metricUnit":"%","id":"graph-classification-mutag","name":"Graph Classification - MUTAG","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":92.06,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","value":88.95,"contributorKind":"method"},{"contributor":"DiffPool","value":85.18,"contributorKind":"method"},{"contributor":"SortPool","value":87.25,"contributorKind":"method"},{"contributor":"Graph2Vec","value":83.68,"contributorKind":"method"},{"contributor":"PATCH_SAN","value":86.12,"contributorKind":"method"},{"contributor":"HGP-SL","value":81.93,"contributorKind":"method"},{"contributor":"DGCNN","value":83.33,"contributorKind":"method"},{"contributor":"SAGPool","value":71.73,"contributorKind":"method"},{"contributor":"DGK","value":85.58,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the IMDB-B dataset.","metricUnit":"%","id":"graph-classification-imdb-b","name":"Graph Classification - IMDB-B","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":76.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","value":74.5,"contributorKind":"method"},{"contributor":"DiffPool","value":72.5,"contributorKind":"method"},{"contributor":"SortPool","value":75.4,"contributorKind":"method"},{"contributor":"Graph2Vec","value":73.9,"contributorKind":"method"},{"contributor":"PATCH_SAN","value":76,"contributorKind":"method"},{"contributor":"HGP-SL","value":74,"contributorKind":"method"},{"contributor":"DGCNN","value":71.6,"contributorKind":"method"},{"contributor":"SAGPool","value":74.8,"contributorKind":"method"},{"contributor":"DGK","value":55,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the IMDB-M dataset.","metricUnit":"%","id":"graph-classification-imdb-m","name":"Graph Classification - IMDB-M","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":51.8,"contributorKind":"method"},{"contributor":"Infograph","value":51.33,"contributorKind":"method"},{"contributor":"DiffPool","value":50.5,"contributorKind":"method"},{"contributor":"SortPool","value":50.47,"contributorKind":"method"},{"contributor":"Graph2Vec","value":52.27,"contributorKind":"method","isCurrentBest":true},{"contributor":"PATCH_SAN","value":46.4,"contributorKind":"method"},{"contributor":"HGP-SL","value":49.53,"contributorKind":"method"},{"contributor":"DGCNN","value":49.2,"contributorKind":"method"},{"contributor":"SAGPool","value":51.33,"contributorKind":"method"},{"contributor":"DGK","value":40.4,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the PROTEINS dataset.","metricUnit":"%","id":"graph-classification-proteins","name":"Graph Classification - PROTEINS","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":75.19,"contributorKind":"method"},{"contributor":"Infograph","value":73.93,"contributorKind":"method"},{"contributor":"DiffPool","value":75.3,"contributorKind":"method"},{"contributor":"SortPool","value":74.48,"contributorKind":"method"},{"contributor":"Graph2Vec","value":73.3,"contributorKind":"method"},{"contributor":"PATCH_SAN","value":75.38,"contributorKind":"method","isCurrentBest":true},{"contributor":"HGP-SL","value":73.94,"contributorKind":"method"},{"contributor":"DGCNN","value":66.75,"contributorKind":"method"},{"contributor":"SAGPool","value":74.03,"contributorKind":"method"},{"contributor":"DGK","value":72.59,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the COLLAB dataset.","metricUnit":"%","id":"graph-classification-collab","name":"Graph Classification - COLLAB","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":79.52,"contributorKind":"method"},{"contributor":"Infograph","value":79.4,"contributorKind":"method"},{"contributor":"DiffPool","value":79.27,"contributorKind":"method"},{"contributor":"SortPool","value":80.07,"contributorKind":"method"},{"contributor":"Graph2Vec","value":85.58,"contributorKind":"method","isCurrentBest":true},{"contributor":"PATCH_SAN","value":74.34,"contributorKind":"method"},{"contributor":"HGP-SL","value":82.08,"contributorKind":"method"},{"contributor":"DGCNN","value":77.45,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the PTC dataset.","metricUnit":"%","id":"graph-classification-ptc","name":"Graph Classification - PTC","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":67.82,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","value":60.74,"contributorKind":"method"},{"contributor":"DiffPool","value":58,"contributorKind":"method"},{"contributor":"SortPool","value":62.04,"contributorKind":"method"},{"contributor":"Graph2Vec","value":54.76,"contributorKind":"method"},{"contributor":"PATCH_SAN","value":61.6,"contributorKind":"method"},{"contributor":"DGCNN","value":56.62,"contributorKind":"method"},{"contributor":"SAGPool","value":59.92,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the NCI1 dataset.","metricUnit":"%","id":"graph-classification-nci1","name":"Graph Classification - NCI1","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":81.66,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","value":76.64,"contributorKind":"method"},{"contributor":"DiffPool","value":69.09,"contributorKind":"method"},{"contributor":"SortPool","value":73.99,"contributorKind":"method"},{"contributor":"Graph2Vec","value":71.85,"contributorKind":"method"},{"contributor":"PATCH_SAN","value":69.82,"contributorKind":"method"},{"contributor":"DGCNN","value":65.96,"contributorKind":"method"},{"contributor":"SAGPool","value":72.87,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Graph classification on the REDDIT-B dataset.","metricUnit":"%","id":"graph-classification-reddit-b","name":"Graph Classification - REDDIT-B","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","value":83.1,"contributorKind":"method"},{"contributor":"Infograph","value":76.55,"contributorKind":"method"},{"contributor":"DiffPool","value":81.2,"contributorKind":"method"},{"contributor":"SortPool","value":78.15,"contributorKind":"method"},{"contributor":"Graph2Vec","value":91.77,"contributorKind":"method","isCurrentBest":true},{"contributor":"PATCH_SAN","value":60.61,"contributorKind":"method"},{"contributor":"DGCNN","value":86.2,"contributorKind":"method"},{"contributor":"SAGPool","value":89.21,"contributorKind":"method"}],"contested":true},{"group":"knowledge-graph-completion","description":"Knowledge graph completion on the FB15k-237 dataset, evaluated using Mean Reciprocal Rank (MRR).","metricUnit":"%","id":"knowledge-graph-completion-fb15k-237","name":"Knowledge Graph Completion - FB15k-237","metricName":"MRR","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"RotatE","value":31.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"ComplEx","value":28.85,"contributorKind":"method"},{"contributor":"TransE","value":30.5,"contributorKind":"method"},{"contributor":"DistMult","value":24.93,"contributorKind":"method"},{"contributor":"CompGCN","value":21.94,"contributorKind":"method"}],"contested":true},{"group":"knowledge-graph-completion","description":"Knowledge graph completion on the WN18 dataset, evaluated using Mean Reciprocal Rank (MRR).","metricUnit":"%","id":"knowledge-graph-completion-wn18","name":"Knowledge Graph Completion - WN18","metricName":"MRR","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"RotatE","value":93.99,"contributorKind":"method","isCurrentBest":true},{"contributor":"ComplEx","value":86.4,"contributorKind":"method"},{"contributor":"TransE","value":71.55,"contributorKind":"method"},{"contributor":"DistMult","value":78.77,"contributorKind":"method"},{"contributor":"CompGCN","value":39.48,"contributorKind":"method"}],"contested":true},{"group":"knowledge-graph-completion","description":"Knowledge graph completion on the WN18RR dataset, evaluated using Mean Reciprocal Rank (MRR).","metricUnit":"%","id":"knowledge-graph-completion-wn18rr","name":"Knowledge Graph Completion - WN18RR","metricName":"MRR","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"RotatE","value":46.05,"contributorKind":"method","isCurrentBest":true},{"contributor":"ComplEx","value":40.77,"contributorKind":"method"},{"contributor":"TransE","value":21.85,"contributorKind":"method"},{"contributor":"DistMult","value":41.64,"contributorKind":"method"},{"contributor":"CompGCN","value":44.8,"contributorKind":"method"}],"contested":true},{"group":"link-prediction","description":"Link prediction on the PPI dataset, evaluated using ROC AUC after removing 10% of edges.","metricUnit":"%","id":"link-prediction-ppi","name":"Link Prediction - PPI","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"ProNE","value":79.93,"contributorKind":"method"},{"contributor":"NetMF","value":79.04,"contributorKind":"method"},{"contributor":"Hope","value":80.21,"contributorKind":"method","isCurrentBest":true},{"contributor":"LINE","value":73.75,"contributorKind":"method"},{"contributor":"Node2vec","value":70.19,"contributorKind":"method"},{"contributor":"NetSMF","value":68.64,"contributorKind":"method"},{"contributor":"DeepWalk","value":69.65,"contributorKind":"method"},{"contributor":"SDNE","value":54.87,"contributorKind":"method"}],"contested":true},{"group":"link-prediction","description":"Link prediction on the Wikipedia dataset, evaluated using ROC AUC after removing 10% of edges.","metricUnit":"%","id":"link-prediction-wikipedia","name":"Link Prediction - Wikipedia","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"ProNE","value":82.74,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetMF","value":73.24,"contributorKind":"method"},{"contributor":"Hope","value":68.89,"contributorKind":"method"},{"contributor":"LINE","value":66.51,"contributorKind":"method"},{"contributor":"Node2vec","value":66.6,"contributorKind":"method"},{"contributor":"NetSMF","value":67.52,"contributorKind":"method"},{"contributor":"DeepWalk","value":65.93,"contributorKind":"method"},{"contributor":"SDNE","value":60.72,"contributorKind":"method"}],"contested":true},{"group":"multiplex-link-prediction","description":"Multiplex link prediction on the Amazon dataset, evaluated using ROC AUC after removing 15% of edges.","metricUnit":"%","id":"multiplex-link-prediction-amazon","name":"Multiplex Link Prediction - Amazon","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GATNE","value":97.44,"contributorKind":"method"},{"contributor":"NetMF","value":97.72,"contributorKind":"method","isCurrentBest":true},{"contributor":"ProNE","value":96.51,"contributorKind":"method"},{"contributor":"Node2vec","value":86.86,"contributorKind":"method"},{"contributor":"DeepWalk","value":92.54,"contributorKind":"method"},{"contributor":"LINE","value":92.56,"contributorKind":"method"},{"contributor":"Hope","value":94.39,"contributorKind":"method"},{"contributor":"GraRep","value":83.88,"contributorKind":"method"}],"contested":true},{"group":"multiplex-link-prediction","description":"Multiplex link prediction on the YouTube dataset, evaluated using ROC AUC after removing 15% of edges.","metricUnit":"%","id":"multiplex-link-prediction-youtube","name":"Multiplex Link Prediction - YouTube","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GATNE","value":84.61,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetMF","value":82.53,"contributorKind":"method"},{"contributor":"ProNE","value":78.96,"contributorKind":"method"},{"contributor":"Node2vec","value":74.01,"contributorKind":"method"},{"contributor":"DeepWalk","value":74.31,"contributorKind":"method"},{"contributor":"LINE","value":73.4,"contributorKind":"method"},{"contributor":"Hope","value":74.66,"contributorKind":"method"},{"contributor":"GraRep","value":71.37,"contributorKind":"method"}],"contested":true},{"group":"multiplex-link-prediction","description":"Multiplex link prediction on the Twitter dataset, evaluated using ROC AUC after removing 15% of edges.","metricUnit":"%","id":"multiplex-link-prediction-twitter","name":"Multiplex Link Prediction - Twitter","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GATNE","value":92.3,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetMF","value":73.75,"contributorKind":"method"},{"contributor":"ProNE","value":81.32,"contributorKind":"method"},{"contributor":"Node2vec","value":78.3,"contributorKind":"method"},{"contributor":"DeepWalk","value":60.29,"contributorKind":"method"},{"contributor":"LINE","value":60.36,"contributorKind":"method"},{"contributor":"Hope","value":70.61,"contributorKind":"method"},{"contributor":"GraRep","value":49.64,"contributorKind":"method"}],"contested":true},{"group":"multiplex-node-classification","description":"Multiplex node classification on the DBLP dataset, evaluated using Macro-F1.","metricUnit":"%","id":"multiplex-node-classification-dblp","name":"Multiplex Node Classification - DBLP","metricName":"Macro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Simple-HGN","value":95.09,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GTN","value":92.03,"contributorKind":"method"},{"contributor":"HAN","value":91.21,"contributorKind":"method"},{"contributor":"GCC","value":79.42,"contributorKind":"method","agent":{"model":"GCC","evidence":"Update GCC result (#409)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"PTE","value":78.65,"contributorKind":"method"},{"contributor":"Metapath2vec","value":75.18,"contributorKind":"method"},{"contributor":"Hin2vec","value":74.31,"contributorKind":"method"}],"contested":true},{"group":"multiplex-node-classification","description":"Multiplex node classification on the ACM dataset, evaluated using Macro-F1.","metricUnit":"%","id":"multiplex-node-classification-acm","name":"Multiplex Node Classification - ACM","metricName":"Macro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Simple-HGN","value":92.57,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GTN","value":90.85,"contributorKind":"method"},{"contributor":"HAN","value":87.25,"contributorKind":"method"},{"contributor":"GCC","value":86.82,"contributorKind":"method","agent":{"model":"GCC","evidence":"Update GCC result (#409)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"PTE","value":87.44,"contributorKind":"method"},{"contributor":"Metapath2vec","value":88.79,"contributorKind":"method"},{"contributor":"Hin2vec","value":84.66,"contributorKind":"method"}],"contested":true},{"group":"multiplex-node-classification","description":"Multiplex node classification on the IMDB dataset, evaluated using Macro-F1.","metricUnit":"%","id":"multiplex-node-classification-imdb","name":"Multiplex Node Classification - IMDB","metricName":"Macro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Simple-HGN","value":58.61,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GTN","value":57.53,"contributorKind":"method"},{"contributor":"HAN","value":53.94,"contributorKind":"method"},{"contributor":"GCC","value":55.86,"contributorKind":"method","agent":{"model":"GCC","evidence":"Update GCC result (#409)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"PTE","value":48.91,"contributorKind":"method"},{"contributor":"Metapath2vec","value":43.1,"contributorKind":"method"},{"contributor":"Hin2vec","value":44.04,"contributorKind":"method"}],"contested":true},{"group":"semi-supervised-node-classification","description":"Semi-supervised node classification under a transductive setting on the Cora dataset.","metricUnit":"%","id":"semi-supervised-node-classification-cora","name":"Semi-Supervised Node Classification - Cora","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Grand","value":84.8,"contributorKind":"method"},{"contributor":"GCNII","value":85.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"DR-GAT","value":83.6,"contributorKind":"method"},{"contributor":"MVGRL","value":83.6,"contributorKind":"method"},{"contributor":"APPNP","value":84.3,"contributorKind":"method"},{"contributor":"Graph U-Net","value":83.3,"contributorKind":"method"},{"contributor":"GAT","value":82.9,"contributorKind":"method"},{"contributor":"GDC_GCN","value":82.5,"contributorKind":"method"},{"contributor":"DropEdge","value":82.1,"contributorKind":"method","agent":{"model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GCN","value":82.3,"contributorKind":"method"},{"contributor":"DGI","value":82,"contributorKind":"method"},{"contributor":"JK-net","value":81.8,"contributorKind":"method","agent":{"model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GraphSAGE","value":80.1,"contributorKind":"method"},{"contributor":"GraphSAGE(unsup)","value":78.2,"contributorKind":"method"},{"contributor":"Chebyshev","value":79,"contributorKind":"method"},{"contributor":"MixHop","value":81.9,"contributorKind":"method"}],"contested":true},{"group":"semi-supervised-node-classification","description":"Semi-supervised node classification under a transductive setting on the Citeseer dataset.","metricUnit":"%","id":"semi-supervised-node-classification-citeseer","name":"Semi-Supervised Node Classification - Citeseer","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Grand","value":75.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"GCNII","value":71.3,"contributorKind":"method"},{"contributor":"DR-GAT","value":72.8,"contributorKind":"method"},{"contributor":"MVGRL","value":73,"contributorKind":"method"},{"contributor":"APPNP","value":72,"contributorKind":"method"},{"contributor":"Graph U-Net","value":71.2,"contributorKind":"method"},{"contributor":"GAT","value":71,"contributorKind":"method"},{"contributor":"GDC_GCN","value":71.2,"contributorKind":"method"},{"contributor":"DropEdge","value":72.1,"contributorKind":"method","agent":{"model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GCN","value":71.4,"contributorKind":"method"},{"contributor":"DGI","value":71.2,"contributorKind":"method"},{"contributor":"JK-net","value":69.5,"contributorKind":"method","agent":{"model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GraphSAGE","value":66.2,"contributorKind":"method"},{"contributor":"GraphSAGE(unsup)","value":65.8,"contributorKind":"method"},{"contributor":"Chebyshev","value":69.8,"contributorKind":"method"},{"contributor":"MixHop","value":71.4,"contributorKind":"method"}],"contested":true},{"group":"semi-supervised-node-classification","description":"Semi-supervised node classification under a transductive setting on the Pubmed dataset.","metricUnit":"%","id":"semi-supervised-node-classification-pubmed","name":"Semi-Supervised Node Classification - Pubmed","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Grand","value":82.4,"contributorKind":"method","isCurrentBest":true},{"contributor":"GCNII","value":80.2,"contributorKind":"method"},{"contributor":"DR-GAT","value":79.1,"contributorKind":"method"},{"contributor":"MVGRL","value":80.1,"contributorKind":"method"},{"contributor":"APPNP","value":80,"contributorKind":"method"},{"contributor":"Graph U-Net","value":79,"contributorKind":"method"},{"contributor":"GAT","value":78.9,"contributorKind":"method"},{"contributor":"GDC_GCN","value":79.8,"contributorKind":"method"},{"contributor":"DropEdge","value":79.7,"contributorKind":"method","agent":{"model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GCN","value":79.5,"contributorKind":"method"},{"contributor":"DGI","value":76.5,"contributorKind":"method"},{"contributor":"JK-net","value":77.7,"contributorKind":"method","agent":{"model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GraphSAGE","value":77.2,"contributorKind":"method"},{"contributor":"GraphSAGE(unsup)","value":78.2,"contributorKind":"method"},{"contributor":"Chebyshev","value":68.6,"contributorKind":"method"},{"contributor":"MixHop","value":80.8,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Unsupervised multi-label node classification on the PPI dataset, evaluated using Micro-F1 score with 90% labels as training data.","metricUnit":"%","id":"unsupervised-node-classification-ppi","name":"Unsupervised Multi-label Node Classification - PPI","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","value":23.73,"contributorKind":"method"},{"contributor":"ProNE","value":24.6,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetSMF","value":23.88,"contributorKind":"method"},{"contributor":"Node2vec","value":20.67,"contributorKind":"method"},{"contributor":"LINE","value":21.82,"contributorKind":"method"},{"contributor":"DeepWalk","value":20.74,"contributorKind":"method"},{"contributor":"Spectral","value":22.48,"contributorKind":"method"},{"contributor":"Hope","value":21.43,"contributorKind":"method"},{"contributor":"GraRep","value":20.6,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Unsupervised multi-label node classification on the Wikipedia dataset, evaluated using Micro-F1 score with 90% labels as training data.","metricUnit":"%","id":"unsupervised-node-classification-wikipedia","name":"Unsupervised Multi-label Node Classification - Wikipedia","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","value":57.42,"contributorKind":"method","isCurrentBest":true},{"contributor":"ProNE","value":56.06,"contributorKind":"method"},{"contributor":"NetSMF","value":53.81,"contributorKind":"method"},{"contributor":"Node2vec","value":54.59,"contributorKind":"method"},{"contributor":"LINE","value":52.46,"contributorKind":"method"},{"contributor":"DeepWalk","value":49.53,"contributorKind":"method"},{"contributor":"Spectral","value":49.35,"contributorKind":"method"},{"contributor":"Hope","value":54.04,"contributorKind":"method"},{"contributor":"GraRep","value":54.37,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Unsupervised multi-label node classification on the Blogcatalog dataset, evaluated using Micro-F1 score with 90% labels as training data.","metricUnit":"%","id":"unsupervised-node-classification-blogcatalog","name":"Unsupervised Multi-label Node Classification - Blogcatalog","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","value":42.47,"contributorKind":"method","isCurrentBest":true},{"contributor":"ProNE","value":41.14,"contributorKind":"method"},{"contributor":"NetSMF","value":40.62,"contributorKind":"method"},{"contributor":"Node2vec","value":40.16,"contributorKind":"method"},{"contributor":"LINE","value":38.06,"contributorKind":"method"},{"contributor":"DeepWalk","value":40.48,"contributorKind":"method"},{"contributor":"Spectral","value":41.41,"contributorKind":"method"},{"contributor":"Hope","value":33.99,"contributorKind":"method"},{"contributor":"GraRep","value":33.48,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Unsupervised multi-label node classification on the DBLP dataset, evaluated using Micro-F1 score with 90% labels as training data.","metricUnit":"%","id":"unsupervised-node-classification-dblp","name":"Unsupervised Multi-label Node Classification - DBLP","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","value":56.72,"contributorKind":"method"},{"contributor":"ProNE","value":56.85,"contributorKind":"method"},{"contributor":"NetSMF","value":59.76,"contributorKind":"method","isCurrentBest":true},{"contributor":"Node2vec","value":57.36,"contributorKind":"method"},{"contributor":"LINE","value":49.78,"contributorKind":"method"},{"contributor":"DeepWalk","value":57.54,"contributorKind":"method"},{"contributor":"Spectral","value":43.68,"contributorKind":"method"},{"contributor":"Hope","value":56.15,"contributorKind":"method"},{"contributor":"GraRep","value":52.76,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Unsupervised multi-label node classification on the Flickr dataset, evaluated using Micro-F1 score with 90% labels as training data.","metricUnit":"%","id":"unsupervised-node-classification-flickr","name":"Unsupervised Multi-label Node Classification - Flickr","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","value":36.27,"contributorKind":"method"},{"contributor":"ProNE","value":36.56,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetSMF","value":35.49,"contributorKind":"method"},{"contributor":"Node2vec","value":36.13,"contributorKind":"method"},{"contributor":"LINE","value":31.61,"contributorKind":"method"},{"contributor":"DeepWalk","value":36.09,"contributorKind":"method"},{"contributor":"Spectral","value":33.09,"contributorKind":"method"},{"contributor":"Hope","value":28.97,"contributorKind":"method"},{"contributor":"GraRep","value":31.83,"contributorKind":"method"}],"contested":true}],"agentStats":[],"repo":{"owner":"THUDM","name":"CogDL","branch":"master"},"participants":[],"stats":{"totalRecords":241,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.133Z"},{"id":"vbench","name":"VBench","tagline":"A comprehensive benchmark suite for evaluating video generative models across multiple dimensions.","description":"VBench decomposes video generation quality into hierarchical dimensions to facilitate fine-grained, objective evaluation. It supports text-to-video, image-to-video, and long video evaluation with automatic metrics highly aligned with human perception. The benchmark helps researchers identify specific strengths and weaknesses in video generation models.","url":"https://github.com/Vchitect/VBench","host":"github","status":"active","domain":"ml-research","tags":["video-generation","text-to-video","image-to-video","evaluation","computer-vision","benchmark"],"images":["https://raw.githubusercontent.com/Vchitect/VBench/master/asset/fig_paper_teaser_vbench.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/asset/fig_extention_teaser.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/VBench-2.0/asset/fig_paper_teaser.jpg"],"participation":{"howToSubmit":"Generate videos using the specified prompt lists, evaluate them using the provided scripts, and submit the resulting JSON file to the Hugging Face Leaderboard or via the Google Form.","submissionFormat":"JSON file containing evaluation results, or a zip file of generated videos/frames.","computeDetails":"Requires GPU for running evaluation models (e.g., DINO, ViCLIP, UMT, CoTracker).","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"automatic","requirements":["Python 3.10+","PyTorch","CUDA","FFmpeg","detectron2"],"compute":"datacenter-gpu"},"links":[{"label":"HuggingFace Leaderboard","url":"https://huggingface.co/spaces/Vchitect/VBench_Leaderboard"},{"label":"VBench Website","url":"https://vchitect.github.io/VBench-project/"},{"label":"VBench 2.0 Website","url":"https://vchitect.github.io/VBench-2.0-project/"},{"label":"VBench Paper","url":"https://arxiv.org/abs/2311.17982"},{"label":"VBench-2.0 Paper","url":"https://arxiv.org/abs/2503.21755"}],"organizer":{"name":"Vchitect","url":"https://vchitect.github.io/VBench-project/","type":"academic"},"venue":{"type":"online"},"quickstart":"pip install vbench\nvbench evaluate --videos_path \"sampled_videos/lavie/human_action\" --dimension \"human_action\"","problems":[{"description":"Evaluation of short video generation models (1.6-4.0s duration, 8-24 FPS) across temporal quality, frame-wise quality, and text alignment.","id":"short-videos","name":"Short Videos Track","metricName":"Total Score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Evaluation of long video generation models (10.0-40.0s duration, 8-24 FPS) across temporal quality, frame-wise quality, text alignment, and long-range subject consistency.","id":"long-videos","name":"Long Videos Track","metricName":"Total Score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Evaluation of advanced video generation capabilities including commonsense reasoning, physics-based realism, human motion, and creative composition.","id":"vbench-2-0","name":"VBench-2.0 (Intrinsic Faithfulness)","metricName":"Total Score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Evaluation of Image-to-Video (I2V) generation models across video-image alignment (subject/background consistency), camera motion, and video quality.","id":"vbench-i2v","name":"VBench-I2V (Image-to-Video)","metricName":"Total Score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"Vchitect","name":"VBench","branch":"master"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.133Z"},{"id":"chess-challenge","name":"Chess Coding Challenge","tagline":"Create a small C# chess bot within a 1024-token limit and compete in a tournament.","description":"A friendly programming competition where participants write a chess bot in C# under a strict 1024-token 'brain capacity' limit. Bots compete in a Swiss tournament followed by a knockout stage to find the most effective and creative implementations.","url":"https://github.com/SebLague/Chess-Challenge","host":"github","status":"ended","domain":"other","tags":["chess","csharp","game-ai","minification","search-algorithms"],"images":[],"participation":{"howToSubmit":"Submissions are now closed. Previously, entries were submitted via a Google Form.","submissionFormat":"A single MyBot.cs file containing the bot implementation.","computeDetails":"The tournament was run by the organizer. Bots were limited to 256MB of memory and had a 5-second limit for constructor execution.","prizes":"Recognition in a results video and inclusion in a game featuring the best bots.","cost":"free","verification":"Automated tournament execution by the organizer.","requirements":["C#",".NET 6.0","Visual Studio or another IDE"],"compute":"none"},"links":[{"label":"Results Video","url":"https://youtu.be/Ne40a5LkK6A"},{"label":"Tournament Data","url":"https://github.com/SebLague/Tiny-Chess-Bot-Challenge-Results"},{"label":"Documentation","url":"https://seblague.github.io/chess-coding-challenge/documentation/"},{"label":"Unofficial Live Leaderboard","url":"https://chess.stjo.dev/"}],"organizer":{"name":"Sebastian Lague","url":"https://github.com/SebLague","type":"individual"},"quickstart":"git clone https://github.com/SebLague/Chess-Challenge.git\ncd Chess-Challenge/Chess-Challenge\ndotnet run","problems":[{"description":"Create a chess bot in C# within a 1024-token limit.","id":"chess-bot","name":"Tiny Chess Bot","metricName":"rank","metricDirection":"minimize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"SebLague","name":"Chess-Challenge","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.133Z"},{"id":"llm-colosseum","name":"LLM Colosseum","tagline":"Evaluate LLMs in real time with Street Fighter III","description":"An interactive benchmark where LLMs play Street Fighter III against each other. Models are evaluated on speed, adaptability, and resilience, with performance tracked via ELO ratings. Submissions are accepted by creating custom robot classes.","url":"https://github.com/OpenGenerativeAI/llm-colosseum","host":"github","status":"active","domain":"llm-eval","tags":["llm-eval","multi-agent","gaming","reinforcement-learning"],"images":["https://raw.githubusercontent.com/OpenGenerativeAI/llm-colosseum/main/notebooks/result_matrix.png","https://github.com/OpenGenerativeAI/llm-colosseum/assets/78322686/3a212601-f54c-490d-aeb9-6f7c2401ebe6"],"participation":{"howToSubmit":"Create a new class inherited from Robot that has the changes you want to make and open a PR.","submissionFormat":"Python class inheriting from Robot in agent/robot.py","verification":"Manual review and evaluation by organizers","requirements":["Python","Docker","DIAMBRA"],"compute":"unknown"},"links":[{"label":"Huggingface Space","url":"https://huggingface.co/spaces/junior-labs/llm-colosseum"},{"label":"Demo","url":"https://llm-colosseum.phospho.ai/"}],"organizer":{"name":"OpenGenerativeAI","type":"community"},"quickstart":"pip install -r requirements.txt\n# Download ROM to ~/.diambra/roms\nmake run","problems":[{"description":"ELO rating of LLMs playing Street Fighter III 3rd Strike.","id":"sf3-elo","name":"Street Fighter III ELO Ranking","metricName":"ELO Rating","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"openai:gpt-4o:text","value":1912.5,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"gpt-4o","family":"openai","confidence":"high","role":"subject"}},{"contributor":"openai:gpt-4o-mini:vision","value":1835.27,"contributorKind":"model","agent":{"model":"gpt-4o-mini","family":"openai","confidence":"high","role":"subject"}},{"contributor":"openai:gpt-4o-mini:text","value":1670.89,"contributorKind":"model","agent":{"model":"gpt-4o-mini","family":"openai","confidence":"high","role":"subject"}},{"contributor":"openai:gpt-4o:vision","value":1656.93,"contributorKind":"model","agent":{"model":"gpt-4o","family":"openai","confidence":"high","role":"subject"}},{"contributor":"mistral:pixtral-large-latest:vision","value":1654.61,"contributorKind":"model","agent":{"model":"pixtral-large-latest","family":"mistral","confidence":"high","role":"subject"}},{"contributor":"mistral:pixtral-12b-2409:vision","value":1590.77,"contributorKind":"model","agent":{"model":"pixtral-12b-2409","family":"mistral","confidence":"high","role":"subject"}},{"contributor":"mistral:pixtral-12b-2409:text","value":1569.03,"contributorKind":"model","agent":{"model":"pixtral-12b-2409","family":"mistral","confidence":"high","role":"subject"}},{"contributor":"together:meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo:text","value":1441.45,"contributorKind":"model","agent":{"model":"Llama-3.2-90B-Vision-Instruct-Turbo","family":"meta","confidence":"high","role":"subject"}},{"contributor":"anthropic:claude-3-haiku-20240307:vision","value":1364.87,"contributorKind":"model","agent":{"model":"claude-3-haiku-20240307","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"mistral:pixtral-large-latest:text","value":1356.32,"contributorKind":"model","agent":{"model":"pixtral-large-latest","family":"mistral","confidence":"high","role":"subject"}},{"contributor":"anthropic:claude-3-haiku-20240307:text","value":1333.6,"contributorKind":"model","agent":{"model":"claude-3-haiku-20240307","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"anthropic:claude-3-sonnet-20240229:vision","value":1314.61,"contributorKind":"model","agent":{"model":"claude-3-sonnet-20240229","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"together:meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo:vision","value":1269.84,"contributorKind":"model","agent":{"model":"Llama-3.2-90B-Vision-Instruct-Turbo","family":"meta","confidence":"high","role":"subject"}},{"contributor":"anthropic:claude-3-sonnet-20240229:text","value":1029.31,"contributorKind":"model","agent":{"model":"claude-3-sonnet-20240229","family":"anthropic","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"OpenGenerativeAI","name":"llm-colosseum","branch":"main"},"participants":[],"stats":{"totalRecords":14,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.133Z"},{"id":"clawprobench","name":"ClawProBench","tagline":"Trace-aware evaluation of AI agents with runtime coverage and frozen workplace-style holdouts.","description":"ClawProBench is a transparent, live-first benchmark harness for evaluating AI agent capabilities inside the OpenClaw runtime. It features deterministic grading across multiple profiles, including core, intelligence, and native tracks. Submissions are evaluated on execution stability and success rates using metrics like FinalScore.","url":"https://github.com/suyoumo/ClawProBench","host":"github","status":"active","domain":"llm-eval","tags":["agent-evaluation","benchmark","openclaw","llm-eval","runtime-coverage"],"images":["https://raw.githubusercontent.com/suyoumo/ClawProBench/main/docs/assets/leaderboard-preview-20260426.png","https://raw.githubusercontent.com/suyoumo/ClawProBench/main/docs/assets/modelpk.png"],"participation":{"howToSubmit":"Run the benchmark locally and email the generated reports to xyh920691910@outlook.com to discuss model evaluation and leaderboard updates.","submissionFormat":"Generated reports in the results/ directory containing avg_score, max_score, coverage-aware summaries, cost, latency, and resume metadata.","computeDetails":"Live runs expect a working local openclaw CLI plus the auth and config required by the surfaces exercised by the selected scenarios.","deadline":"rolling","cost":"free","verification":"Deterministic grading with custom checkers and order-insensitive matchers inside the OpenClaw runtime.","requirements":["Python 3.11","uv","OpenClaw CLI","Docker"],"compute":"unknown"},"links":[{"label":"Leaderboard Website","url":"https://suyoumo.github.io/bench/"},{"label":"arXiv Paper","url":"https://arxiv.org/abs/2608.22510"},{"label":"Hugging Face Dataset","url":"https://huggingface.co/datasets/xyh110sym/clawprobench"}],"organizer":{"name":"Xiao YuanHang","type":"individual"},"quickstart":"pip install uv\nuv venv --python 3.11\nsource .venv/bin/activate\nuv pip install -r requirements.txt\npython3 run.py inventory\npython3 run.py run --model '<MODEL>' --execution-mode live --benchmark-profile core --trials 1 --cleanup-agents","problems":[{"description":"Default ranking suite consisting of 26 active scenarios.","id":"core","name":"Core Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Extended active capability benchmark consisting of 95 active scenarios.","id":"intelligence","name":"Intelligence Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Lower-stakes breadth and regression slice consisting of 7 active scenarios.","id":"coverage","name":"Coverage Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Active OpenClaw-native slice only, consisting of 36 active scenarios.","id":"native","name":"Native Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Union of all active scenarios, consisting of 102 active scenarios.","id":"full","name":"Full Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"suyoumo","name":"ClawProBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"squad-explorer","name":"SQuAD","tagline":"The Stanford Question Answering Dataset benchmark for reading comprehension.","description":"SQuAD is a reading comprehension dataset consisting of questions posed by crowdworkers on a set of Wikipedia articles. Models must find the answer span or determine if the question is unanswerable. It is a premier benchmark for question answering systems.","url":"https://rajpurkar.github.io/SQuAD-explorer/","host":"website","status":"active","domain":"ml-research","tags":["nlp","question-answering","reading-comprehension","dataset"],"images":[],"participation":{"howToSubmit":"Submissions are made by evaluating models on CodaLab Worksheets and submitting the worksheet to the organizers.","submissionFormat":"CodaLab worksheet","cost":"free","requirements":["Python","CodaLab"],"compute":"unknown"},"links":[{"label":"Website","url":"https://rajpurkar.github.io/SQuAD-explorer/"},{"label":"GitHub","url":"https://github.com/rajpurkar/SQuAD-explorer"}],"organizer":{"name":"Stanford NLP Group","url":"https://nlp.stanford.edu/","type":"academic"},"problems":[{"description":"The initial version of the Stanford Question Answering Dataset, containing 100,000+ question-answer pairs where the answer is a segment of text.","id":"squad-1-1","name":"SQuAD 1.1","metricName":"F1","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers.","id":"squad-2-0","name":"SQuAD 2.0","metricName":"F1","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"rajpurkar","name":"SQuAD-explorer","branch":"master"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"polymarket-paper-trader","name":"Polymarket Paper Trader","tagline":"A zero-risk gym for AI agents to practice, evaluate, and benchmark decision intelligence on real prediction markets.","description":"A faithful paper-trading environment and evaluation harness for AI agents on Polymarket. Agents trade with simulated cash against live order books with official fee models. It features the Forecast Arena, a daily public forecasting leaderboard, and a paper-trading competition ranked by ROI.","url":"https://github.com/agent-next/polymarket-paper-trader","host":"github","status":"active","domain":"llm-eval","tags":["prediction-markets","paper-trading","llm-evaluation","decision-intelligence","mcp-server","polymarket"],"images":[],"participation":{"howToSubmit":"For Forecast Arena, add your model to arena.yaml. For Paper Trading, generate a leaderboard entry using the CLI or MCP tool and open a GitHub issue with the JSON.","submissionFormat":"YAML configuration for Forecast Arena; JSON output from leaderboard_entry for Paper Trading.","computeDetails":"Forecast Arena runs daily on GitHub Actions CI. Local evaluations run on your own compute.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"Forecast Arena forecasts are verified against real Polymarket outcomes. Paper trading submissions are generated as verifiable JSON.","requirements":["Python 3.10+","pip","litellm"],"compute":"none"},"links":[{"label":"Forecast Arena Leaderboard","url":"https://polymarket-leaderboard.com/"},{"label":"GitHub Repository","url":"https://github.com/agent-next/polymarket-paper-trader"}],"organizer":{"name":"agent-next","url":"https://github.com/agent-next","type":"community"},"quickstart":"pip install polymarket-paper-trader\npm-trader init --balance 10000\npm-trader buy will-bitcoin-hit-100k yes 100\npm-trader stats","problems":[{"description":"AI models and baselines forecast soon-resolving Polymarket binary markets daily without knowing the market price, scored by Brier score against real outcomes.","id":"forecast-arena","name":"Forecast Arena","metricName":"Brier score","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"AI agents trade $10,000 of paper money against live Polymarket order books, ranked by ROI% with Sharpe ratio as a tiebreaker.","metricUnit":"%","id":"paper-trading","name":"Paper Trading","metricName":"ROI","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"agent-next","name":"polymarket-paper-trader","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"skab","name":"SKAB","tagline":"Benchmark for evaluating anomaly detection algorithms on multivariate time series data.","description":"The Skoltech Anomaly Benchmark (SKAB) is designed for evaluating anomaly detection algorithms on multivariate time series data from a physical testbed. It supports outlier detection (single-point anomalies) and changepoint detection (collective anomalies). Submissions are evaluated using F1-score and NAB metrics.","url":"https://github.com/waico/SKAB","host":"github","status":"active","domain":"ml-research","tags":["anomaly-detection","time-series","benchmark","outlier-detection","changepoint-detection"],"images":[],"participation":{"howToSubmit":"Evaluate your algorithm using the SKAB datasets and submit results on Kaggle or PapersWithCode.","submissionFormat":"Kaggle submission or PapersWithCode entry","cost":"free","requirements":["Python 3.10+","Poetry"],"compute":"unknown"},"links":[{"label":"Kaggle Dataset","url":"https://www.kaggle.com/yuriykatser/skoltech-anomaly-benchmark-skab"},{"label":"PapersWithCode CPD","url":"https://paperswithcode.com/sota/change-point-detection-on-skab"},{"label":"Medium Article on Metrics","url":"https://medium.com/@katser/a-review-of-anomaly-detection-metrics-with-a-lot-of-related-information-736d88774712"}],"organizer":{"name":"Skoltech","url":"https://www.skoltech.ru/en","type":"academic"},"quickstart":"git clone https://github.com/waico/SKAB.git\ncd SKAB\npoetry install","problems":[{"description":"Anomalies are considered and marked up as single-point anomalies. Evaluated as a binary classification problem.","id":"outlier-detection","name":"Outlier Detection","metricName":"F1","metricDirection":"maximize","baseline":0,"isOpen":false,"records":[{"contributor":"Perfect detector","description":"Theoretical upper bound","value":1,"contributorKind":"method"},{"contributor":"Conv-AE","value":0.78,"contributorKind":"method","isCurrentBest":true},{"contributor":"MSET","value":0.78,"contributorKind":"method"},{"contributor":"T-squared+Q (PCA-based)","value":0.76,"contributorKind":"method"},{"contributor":"LSTM-AE","value":0.74,"contributorKind":"method"},{"contributor":"T-squared","value":0.66,"contributorKind":"method"},{"contributor":"LSTM-VAE","value":0.56,"contributorKind":"method"},{"contributor":"Vanilla LSTM","value":0.54,"contributorKind":"method"},{"contributor":"Vanilla AE","value":0.39,"contributorKind":"method"},{"contributor":"MSCRED","value":0.36,"contributorKind":"method"},{"contributor":"Isolation forest","value":0.29,"contributorKind":"method"},{"contributor":"Null detector","value":0,"contributorKind":"method","isBaseline":true}],"contested":true},{"description":"Anomalies are considered and marked up as collective anomalies. Evaluated using NAB (Numenta Anomaly Benchmark) metrics with a 60-second window.","id":"changepoint-detection","name":"Changepoint Detection","metricName":"NAB (standard)","metricDirection":"maximize","baseline":0,"isOpen":false,"records":[{"contributor":"Perfect detector","description":"Theoretical upper bound","value":100,"contributorKind":"method"},{"contributor":"MSCRED","value":32.42,"contributorKind":"method","isCurrentBest":true},{"contributor":"Isolation forest","value":26.16,"contributorKind":"method"},{"contributor":"T-squared+Q (PCA-based)","value":25.35,"contributorKind":"method"},{"contributor":"Conv-AE","value":23.61,"contributorKind":"method"},{"contributor":"LSTM-AE","value":23.51,"contributorKind":"method"},{"contributor":"T-squared","value":19.54,"contributorKind":"method"},{"contributor":"MSET","value":13.84,"contributorKind":"method"},{"contributor":"Vanilla AE","value":11.41,"contributorKind":"method"},{"contributor":"Vanilla LSTM","value":11.31,"contributorKind":"method"},{"contributor":"ArimaFD","value":-0.09,"contributorKind":"method"},{"contributor":"Null detector","value":0,"contributorKind":"method","isBaseline":true}],"contested":true}],"agentStats":[],"repo":{"owner":"waico","name":"SKAB","branch":"master"},"participants":[],"stats":{"totalRecords":24,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"putnambench","name":"PutnamBench","tagline":"Multilingual benchmark for evaluating neural theorem-provers on the Putnam Mathematical Competition.","description":"PutnamBench evaluates theorem-proving algorithms on competition mathematics problems from the William Lowell Putnam Mathematical Competition (1962-2025). Supporting Lean 4, Isabelle, and Coq, it features tasks for both proof-only and proof-with-answer generation. Submissions are evaluated and tracked on an official leaderboard.","url":"https://github.com/trishullab/PutnamBench","host":"github","status":"active","domain":"formal-methods","tags":["theorem-proving","formal-methods","lean4","isabelle","coq","math-olympiad"],"images":[],"participation":{"howToSubmit":"Reach out privately at george.tsoukalas@utexas.edu with requests for additions to the leaderboard accompanied by a preprint or publication.","submissionFormat":"Evaluation results accompanied by a preprint or publication. Do not share proofs publicly.","cost":"free","verification":"human review","requirements":["Lean 4","Isabelle","Coq","Python"],"compute":"unknown"},"links":[{"label":"Leaderboard","url":"https://trishullab.github.io/PutnamBench/leaderboard.html"},{"label":"Website","url":"https://trishullab.github.io/PutnamBench/"},{"label":"Paper","url":"https://arxiv.org/abs/2407.11214"}],"organizer":{"name":"Trishul Lab","url":"https://github.com/trishullab","type":"academic"},"problems":[{"description":"Proving the problem with the numerical answer written into the theorem statement using Isabelle.","id":"isabelle-proof-only","name":"Isabelle (Proof Only)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Producing the numerical answer along with the proof of correctness using Isabelle.","id":"isabelle-with-answer","name":"Isabelle (With Answer)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Proving the problem with the numerical answer written into the theorem statement using Coq.","id":"coq-proof-only","name":"Coq (Proof Only)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Producing the numerical answer along with the proof of correctness using Coq.","id":"coq-with-answer","name":"Coq (With Answer)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Proving the problem with the numerical answer written into the theorem statement using Lean 4.","id":"lean4-proof-only","name":"Lean 4 (Proof Only)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Producing the numerical answer along with the proof of correctness using Lean 4.","id":"lean4-with-answer","name":"Lean 4 (With Answer)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"trishullab","name":"PutnamBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","tagline":"Benchmark for AI decision models measuring accuracy, probability calibration, latency, and cost.","description":"JevBench evaluates AI decision models (including Jev-compatible systems) on typed decision accuracy, probability calibration, latency, and cost. It features multiple tiers of difficulty, including a 220-decision hard tier, and is maintained independently by Benchmark Heaven.","url":"https://github.com/fstandhartinger/jevbench","host":"github","status":"active","domain":"llm-eval","tags":["benchmark","decision-models","llm-evaluation","calibration","accuracy"],"images":[],"participation":{"howToSubmit":"Submit an issue on the GitHub repository with your model endpoint, serving code, or repository details.","submissionFormat":"Model endpoint, serving code, or Hugging Face repository.","computeDetails":"Evaluations are run by the organizer on rented RunPod GPUs (e.g., RTX PRO 4500 Blackwell, H100 NVL) or local CPU.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"The organizer runs the evaluation harness against the submitted endpoint or model weights.","requirements":["Python 3.10+"],"compute":"none"},"links":[{"label":"Live Leaderboard","url":"https://benchmarkheaven.com/jev-models"},{"label":"Methodology","url":"https://benchmarkheaven.com/jev-models/methodology"},{"label":"Data Card","url":"https://benchmarkheaven.com/jev-models/data"}],"organizer":{"name":"Benchmark Heaven","url":"https://benchmarkheaven.com/","type":"individual"},"quickstart":"python -m unittest discover -s tests -v\npython -m jevbench.cli run --tasks datasets/public/original.jsonl \\\n  --adapter typesafe --model jev-latest --key-env TYPESAFE_API_KEY \\\n  --price-in-per-m 0.042 --price-out-per-m 0 \\\n  --results RUN/results.jsonl --raw-dir RUN/raw \\\n  --ledger RUN/ledger.jsonl --cap-usd 15 --manifest RUN/manifest.json","problems":[{"description":"Historical release evaluating models on the v1.4 protocol with one rotation and calibration.","id":"jevbench-v1.4.2.2","name":"JevBench v1.4.2.2","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Imajev-4B","value":67.37,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","value":65.84,"contributorKind":"model","agent":{"model":"JevK5 v0.2 + LoRA","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"decider-4b v2 (Mapika)","value":64.13,"contributorKind":"model","agent":{"model":"decider-4b v2 (Mapika)","evidence":"recomputes JevBench scores, preset scores, ranks, intelligence, speed and cost axes of the v1.4.2.1 and v1.4.2.2 results","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":63.29,"contributorKind":"model","agent":{"model":"Jev 1.13.0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"JevK5 v0.2.0","value":62.04,"contributorKind":"model","agent":{"model":"JevK5 v0.2.0","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Evaluates models on 534 decisions across easy, standard, judge, and hard tiers. Score is the geometric mean of Intelligence, Calibration, Speed, and Cost.","id":"jevbench-v1.3.0","name":"JevBench v1.3.0 (v1.2 results)","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jev 1.13.0","value":74.4,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Jev 1.13.0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"SemIf (Qwen3.5-4B)","value":73.1,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"djev (Maisa, diffusion-gemma)","value":73,"contributorKind":"model","agent":{"model":"DiffusionGemma","family":"google","confidence":"high","role":"subject"}},{"contributor":"Winnow-12B Q8","value":71.2,"contributorKind":"model","agent":{"model":"Gemma 3 12B","family":"google","confidence":"high","role":"subject"}},{"contributor":"reflex 4B (kshetrajna12)","value":70.3,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"jqv (Qwen3-32B zero-shot)","value":68.6,"contributorKind":"model","agent":{"model":"Qwen3-32B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"decision-machine-1 (milliseconds.ai)","value":68.3,"contributorKind":"model","agent":{"model":"decision-machine-1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"decider-35b-a3b (Mapika)","value":67.6,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","value":67,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"system-one-open (Gemma 4 E2B LoRA on an L4)","value":66.6,"contributorKind":"model","agent":{"model":"Gemma 4 E2B","family":"google","confidence":"high","role":"subject"}},{"contributor":"OpenJev razorback16 (DiffusionGemma 26B)","value":66.4,"contributorKind":"model","agent":{"model":"DiffusionGemma 26B","family":"google","confidence":"high","role":"subject"}},{"contributor":"SimpleJev Qwen3.8-27B","value":66.3,"contributorKind":"model","agent":{"model":"Qwen3.8-27B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"ZeroEntropy zerank-2","value":66,"contributorKind":"model","agent":{"model":"zerank-2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Luna (low)","value":65.9,"contributorKind":"model","agent":{"model":"GPT-5.6 Luna","family":"openai","confidence":"high","role":"subject"}},{"contributor":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","value":65.3,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Qwen3-Reranker-4B","value":63.8,"contributorKind":"model","agent":{"model":"Qwen3-Reranker-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"reflex-27b (Qwen3.8-27B)","value":63.3,"contributorKind":"model","agent":{"model":"Qwen3.8-27B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"LitJev (Qwen3.8-27B)","value":62.7,"contributorKind":"model","agent":{"model":"Qwen3.8-27B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"kev 0.6B (research preview)","value":62.5,"contributorKind":"model","agent":{"model":"kev 0.6B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"SimpleJev Qwen3.6-35B-A3B","value":62.5,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"djev (thinking)","value":62.4,"contributorKind":"model","agent":{"model":"DiffusionGemma","family":"google","confidence":"high","role":"subject"}},{"contributor":"jev-local (Qwen3.5-9B)","value":61.8,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"decider-2b (Mapika)","value":61.7,"contributorKind":"model","agent":{"model":"Qwen3.5-2B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Bespoke Nimble 9B","value":60.5,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Flash-Lite","value":60.1,"contributorKind":"model","agent":{"model":"Gemini 3.1 Flash-Lite","family":"google","confidence":"high","role":"subject"}},{"contributor":"OpenJev (thinking, BF16)","value":60,"contributorKind":"model","agent":{"model":"DiffusionGemma 26B","family":"google","confidence":"high","role":"subject"}},{"contributor":"kev 4B (research preview)","value":59.7,"contributorKind":"model","agent":{"model":"kev 4B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4.1 Flash","value":57.5,"contributorKind":"model","agent":{"model":"DeepSeek V4.1 Flash","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"kev 8B (research preview)","value":56.4,"contributorKind":"model","agent":{"model":"kev 8B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Open-Jev 9B (Zefan Cai)","value":55,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"system-one (Qwen3-8B, Goedecke)","value":54.8,"contributorKind":"model","agent":{"model":"Qwen3-8B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"jeff (GLiFormer 400M)","value":54.4,"contributorKind":"model","agent":{"model":"GLiFormer 400M","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Laya (421M)","value":54.4,"contributorKind":"model","agent":{"model":"ModernBERT-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Open-Jev 2B (Zefan Cai)","value":51.3,"contributorKind":"model","agent":{"model":"Qwen3.5-2B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"OpenDecision (ModernBERT-large zero-shot)","value":40.6,"contributorKind":"model","agent":{"model":"ModernBERT-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openJev Verdict 1.4","value":38.9,"contributorKind":"model","agent":{"model":"openJev Verdict 1.4","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openJev Verdict (151M)","value":38.1,"contributorKind":"model","agent":{"model":"openJev Verdict 151M","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"kev 0.5B","value":33.2,"contributorKind":"model","agent":{"model":"kev 0.5B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2 large (Fastino)","value":29.6,"contributorKind":"model","agent":{"model":"GLiNER2 large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"smalljev semantic-v9","value":27.4,"contributorKind":"model","agent":{"model":"MiniCPM5-2B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2 (gliner2.5-base)","value":24,"contributorKind":"model","agent":{"model":"gliner2.5-base","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"open-jev-deberta-v3-large (local CPU)","value":23.1,"contributorKind":"model","agent":{"model":"DeBERTa-v3-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2.5 multi (Fastino, 287M)","value":16.6,"contributorKind":"model","agent":{"model":"GLiNER2.5 multi","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2.5 small (Fastino, 74M)","value":13.8,"contributorKind":"model","agent":{"model":"GLiNER2.5 small","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Mixedbread mxbai-rerank-base-v2","value":0.8,"contributorKind":"model","agent":{"model":"mxbai-rerank-base-v2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"BAAI bge-reranker-v2-m3","value":0.7,"contributorKind":"model","agent":{"model":"bge-reranker-v2-m3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Alibaba GTE Reranker ModernBERT-base","value":0.3,"contributorKind":"model","agent":{"model":"GTE Reranker ModernBERT-base","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Certo v1 (AltSlate Labs)","value":0,"contributorKind":"model","agent":{"model":"Certo v1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"classifier.dev (fast tier)","value":83.6,"contributorKind":"method","isBaseline":false,"agent":{"model":"Jev 1.13.0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Qwen3.8 27B (partial run)","value":24.8,"contributorKind":"model","agent":{"model":"Qwen3.8 27B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Needle 3, options as tools (partial run)","value":1.1,"contributorKind":"model","agent":{"model":"Needle 3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Needle 3 (partial run)","value":0.1,"contributorKind":"model","agent":{"model":"Needle 3","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Evaluates models on the frozen v1.1 task set (314 typed decisions) with Balanced 33:33:33 scoring, including open rebuilds run on rented RunPod GPUs.","id":"jevbench-v1.1.3","name":"JevBench v1.1.3 (The GPU Round)","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ) (GPU)","value":83.3,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev, author's yes/no order* (post-hoc adapter mode) (GPU)","value":79.7,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16) (GPU)","value":78.2,"contributorKind":"model","agent":{"model":"DiffusionGemma 26B","family":"google","confidence":"high","role":"subject"}},{"contributor":"system-one (Qwen3-8B, Sean Goedecke) (GPU)","value":77.9,"contributorKind":"model","agent":{"model":"Qwen3-8B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev (Qwen3.5-4B, IkerMoel) (GPU)","value":76.8,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"system-one-open (Gemma 4 E2B LoRA on an L4)","value":75.7,"contributorKind":"model","agent":{"model":"Gemma 4 E2B","family":"google","confidence":"high","role":"subject"}},{"contributor":"Bespoke Nimble 9B (Bespoke Labs) (GPU)","value":73.8,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":73.6,"contributorKind":"model","agent":{"model":"Jev 1.13.0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","value":69.6,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Flash-Lite","value":65,"contributorKind":"model","agent":{"model":"Gemini 3.1 Flash-Lite","family":"google","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Luna (low reasoning effort)","value":62.2,"contributorKind":"model","agent":{"model":"GPT-5.6 Luna","family":"openai","confidence":"high","role":"subject"}},{"contributor":"open-jev-deberta-v3-large (local CPU)","value":60.6,"contributorKind":"model","agent":{"model":"DeBERTa-v3-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4.1 Flash (thinking default)","value":55.7,"contributorKind":"model","agent":{"model":"DeepSeek V4.1 Flash","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Needle 3, options as tools (post-hoc adapter mode)","value":41.5,"contributorKind":"model","agent":{"model":"Needle 3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Needle 3 (Cactus, 2-bit, local CPU)","value":40.3,"contributorKind":"model","agent":{"model":"Needle 3","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"fstandhartinger","name":"jevbench","branch":"main"},"participants":[],"stats":{"totalRecords":72,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"lora-speedrun","name":"LoRA Speedrun","tagline":"A wall-clock speedrun benchmark for LoRA fine-tuning on a single L40S GPU.","description":"LoRA Speedrun is a wall-clock optimization competition for low-rank adaptation fine-tuning. Participants optimize training speed on frozen tasks and hardware (a single NVIDIA L40S) while meeting target accuracy thresholds. Submissions are verified by running three times with fresh seeds in a sandboxed environment.","url":"https://github.com/Saivineeth147/lora-speedrun","host":"github","status":"active","domain":"systems-perf","tags":["lora","fine-tuning","llm","speedrun","optimization","pytorch"],"images":["https://raw.githubusercontent.com/Saivineeth147/lora-speedrun/main/docs/leaderboard.png"],"participation":{"howToSubmit":"Copy submissions/TEMPLATE/ to submissions/NNN-<your-github-handle>/, implement train.py, config.yaml, and NOTES.md, and open a PR.","submissionFormat":"A directory containing train.py, config.yaml, and NOTES.md.","computeDetails":"1× NVIDIA L40S (48 GB) via Modal network-blocked sandbox","deadline":"rolling","prizes":"recognition only","eligibility":"open to everyone","cost":"free","verification":"Automated CI validation, AI security screen, and 3x sandboxed reruns on Modal L40S with fresh seeds.","requirements":["Python","PyTorch","Modal","CUDA"],"compute":"datacenter-gpu"},"links":[{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/vineeth98/lora-speedrun"}],"organizer":{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","type":"individual"},"venue":{"type":"online"},"quickstart":"git clone https://github.com/Saivineeth147/lora-speedrun && cd lora-speedrun\npip install modal pyyaml && modal setup\npython harness/modal_verify.py --prefetch\npython harness/modal_verify.py --submission submissions/000-baseline --runs 1","problems":[{"description":"Fine-tune Qwen2.5-1.5B on GSM8K train split to achieve >= 57.0% exact-match accuracy as fast as possible.","metricUnit":"s","id":"t1","name":"Track 1 — GSM8K · Qwen2.5-1.5B · target ≥ 57.0% · 1× L40S","metricName":"time","metricDirection":"minimize","baseline":716.9,"isOpen":false,"records":[{"contributor":"Saivineeth147","description":"Baseline: plain LoRA r=16 on all linear layers, 3 epochs, cosine LR. No tricks.","date":"2026-07-18","value":716.9,"contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/000-baseline"],"isBaseline":true,"isCurrentBest":false,"agent":{"model":"Claude Fable 5","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"Saivineeth147","description":"Sequence packing + completion-only loss masking, 2 epochs. Same LoRA config as #0; ~2x faster at higher accuracy.","date":"2026-07-18","value":364.9,"contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/001-saivineeth147"],"isBaseline":false,"isCurrentBest":false,"agent":{"model":"Claude Fable 5","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"stared","description":"One 4e-4 epoch over 3,000 examples with 2x optimizer updates (integrated-LR), 512-token packs. First outside record.","date":"2026-07-20","value":113,"contributorKind":"person","contributorUrl":"https://github.com/stared","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/002-pmigdal"],"isBaseline":false,"isCurrentBest":false,"agent":{"model":"Claude Fable","evidence":"branch: migdal-fable","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"stared","description":"Shortest-4k data pruning, 1 aggressive-LR epoch, <<...>> annotations stripped, custom GPU-resident packed loop, chunked completion-only CE (no full logits).","date":"2026-07-20","value":104,"contributorKind":"person","contributorUrl":"https://github.com/stared","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/003-pmigdal"],"isBaseline":false,"isCurrentBest":false,"agent":{"model":"Claude Fable 5","evidence":"PR branch name 'migdal-fable'","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"slippylolo","description":"0.75-epoch six-target BF16 LoRA: shortest-4k pruning with down_proj dropped (13.76M params), deterministic best-fit packing, per-layer fused base+LoRA-A q/k/v & gate/up GEMMs, chunked completion-only CE.","date":"2026-07-25","value":43.9,"contributorKind":"person","contributorUrl":"https://github.com/slippylolo","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/004-slippylolo"],"isBaseline":false,"isCurrentBest":false},{"contributor":"slippylolo","description":"Direct-from-safetensors Qwen load with a warmed page cache, then 12 optimizer updates: 8 full-network, then 4 through only the top decoder layer over a cached BF16 frozen-prefix boundary.","date":"2026-08-03","value":21.6,"contributorKind":"person","contributorUrl":"https://github.com/slippylolo","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/005-slippylolo"],"isBaseline":false,"isCurrentBest":true}],"contested":true},{"description":"Fine-tune SmolLM2-1.7B on SQuAD v1.1 train split to achieve >= 75.5% exact-match accuracy as fast as possible.","metricUnit":"s","id":"t2","name":"Track 2 — SQuAD v1.1 · SmolLM2-1.7B · target ≥ 75.5% · 1× L40S","metricName":"time","metricDirection":"minimize","baseline":525.4,"isOpen":false,"records":[{"contributor":"Saivineeth147","description":"Track 2 baseline: plain LoRA r=16 on SQuAD, first 20k examples, 1 epoch, full-sequence loss. No tricks.","date":"2026-07-20","value":525.4,"contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/t2-000-baseline"],"isBaseline":true,"isCurrentBest":false,"agent":{"model":"Claude Fable 5","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"abacon","description":"Track 2 port of the Track 1 stack: shortest-4k pruning, best-fit packing, fused base+LoRA-A GEMMs, chunked completion-only CE, 0.75 epoch on SmolLM2-1.7B.","date":"2026-07-27","value":49.5,"contributorKind":"person","contributorUrl":"https://github.com/abacon","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/t2-001-bacon"],"isBaseline":false,"isCurrentBest":true}],"contested":false}],"agentStats":[{"family":"anthropic","records":3,"currentBests":0,"models":["Claude Fable 5","Claude Fable"],"contributors":["Saivineeth147","stared"]}],"repo":{"owner":"Saivineeth147","name":"lora-speedrun","branch":"main"},"participants":[{"name":"slippylolo","url":"https://github.com/slippylolo","submissions":2,"problems":["t1"],"lastActive":"2026-08-03","bestRank":1},{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","submissions":3,"problems":["t1","t2"],"lastActive":"2026-07-20"},{"name":"stared","url":"https://github.com/stared","submissions":2,"problems":["t1"],"lastActive":"2026-07-20"},{"name":"abacon","url":"https://github.com/abacon","submissions":1,"problems":["t2"],"lastActive":"2026-07-27"}],"stats":{"totalRecords":8,"uniqueParticipants":4,"firstSubmission":"2026-07-18","lastSubmission":"2026-08-03","recordsLast30d":0,"recordsLast90d":8},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"equational-theories-lean-stage2","name":"Mathematics Distillation Challenge — Equational Theories — Stage 2","tagline":"Machine-verifiable Lean 4 certificates for equational implication over magmas.","description":"A competition challenging participants to decide equational implications over magmas. Submissions must provide machine-verifiable Lean 4 proofs or countermodels, which are evaluated by a deterministic judge. It features two tracks: Solo (fixed per-problem budget) and Marathon (shared global budget).","url":"https://github.com/SAIRcompetition/equational-theories-lean-stage2","host":"github","status":"active","domain":"formal-methods","tags":["lean4","theorem-proving","magmas","formal-verification","math-olympiad"],"images":[],"participation":{"howToSubmit":"Submit a single solver.py file (≤ 500 KB) implementing the track's I/O protocol.","submissionFormat":"A single Python file solver.py","computeDetails":"Docker sandbox with 2 CPUs and 2048 MB RAM in production","cost":"free","verification":"Deterministic Lean verifier (judge/verify.py) checking proofs/countermodels","requirements":["Python 3.8+","Lean 4","elan","Mathlib"],"compute":"cpu"},"links":[{"label":"Official Competition Page","url":"https://competition.sair.foundation/competitions/mathematics-distillation-challenge-equational-theories-stage2/overview"},{"label":"Equational Theories Project","url":"https://teorth.github.io/equational_theories/"},{"label":"HuggingFace Dataset","url":"https://huggingface.co/datasets/SAIRfoundation/equational-theories-selected-problems"}],"organizer":{"name":"SAIR Foundation","url":"https://competition.sair.foundation/","type":"foundation"},"quickstart":"bash scripts/setup.sh\npip install openai\nsource .env.judge\npython3 scripts/run_harness.py\npython3 -m pipeline.runner --submission examples/solo/demos/baseline --problems examples/problems/sample_20.json","problems":[{"description":"One problem per solver subprocess with a fixed per-problem budget of 3600s.","id":"solo-track","name":"Solo Track","metricName":"accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"N problems per solver subprocess with a shared global budget of N x 300s.","id":"marathon-track","name":"Marathon Track","metricName":"accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"SAIRcompetition","name":"equational-theories-lean-stage2","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.134Z"},{"id":"first-proof","name":"First Proof","tagline":"Unpublished research-level math problems; AI systems get one shot each and expert referees grade the proofs blind.","description":"First Proof is an independent benchmark run by research mathematicians to measure whether AI systems can autonomously prove research-level results. Each batch collects problems that arose in the contributors' own work and whose proofs are unpublished; the editorial board runs every system itself, one shot per problem with no human input, and expert referees grade the proofs double blind. Batch 2 (2026-06-10): 4 systems, 39 graded solutions, 7 of 10 problems got at least one passing proof; P4, P8, P10 remain open. Batch 3 is under way; results are due 2026-10-14.","url":"https://1stproof.org/","host":"website","status":"active","domain":"llm-eval","tags":["mathematics","research-math","proofs","referee-graded","ai-evaluation","benchmark"],"images":[],"participation":{"howToSubmit":"Systems enter per batch through the call for submissions on 1stproof.org (Batch 3 applications closed 2026-09-10). A system is an open-source harness calling publicly available models, or a public product; First Proof clones the submitter's repository, builds its Docker image on an approved EC2 instance and runs it once on the problem file, following the protocol published with each batch (e.g. 1stproof/batch-2).","submissionFormat":"A repository with Dockerfile, hardware.json, README.md and src/; the container reads the LaTeX problems from /data/input/input.json and writes LaTeX proofs to /data/output.","computeDetails":"Runs on First Proof's AWS instances; model API usage goes on the submitter's keys. Batch 2 systems spent $117–$4,799 in API fees for 10 problems.","deadline":"Batch 3: applications until 2026-09-10; results 2026-10-14.","rounds":"Batches of unpublished research problems, each graded and published with every solution and referee report, plus community experiments where problems are posted publicly and solutions follow a week later.","prizes":"recognition only","eligibility":"AI companies and academic teams; only publicly available models, or harnesses that will be released.","cost":"free to enter; API usage is paid by the submitter","verification":"Each proof is refereed double blind by at least two experts in the field; the editorial board decides essentially flawless, minor revisions, major revisions or reject. Problems, human solutions, AI solutions, referee reports and logs are all published.","requirements":["Open-source harness or public AI product","Publicly available models","Docker","Model API keys"],"compute":"none"},"links":[{"label":"Website","url":"https://1stproof.org/"},{"label":"Batch 3 (results 2026-10-14)","url":"https://1stproof.org/third-batch.html"},{"label":"Batch 2 report (arXiv:2606.18119)","url":"https://arxiv.org/abs/2606.18119"},{"label":"Batch 2: solutions, reviews, harnesses","url":"https://github.com/1stproof/batch-2"},{"label":"Batch 1 paper (arXiv:2602.05192)","url":"https://arxiv.org/abs/2602.05192"},{"label":"Batch 1 problems (1stproof/batch-1)","url":"https://github.com/1stproof/batch-1"},{"label":"Community experiment","url":"https://1stproof.org/community-experiment.html"}],"organizer":{"name":"First Proof Foundation","url":"https://1stproof.org/","type":"foundation"},"venue":{"location":"1stproof.org / GitHub","type":"online"},"quickstart":"git clone https://github.com/1stproof/batch-2 && cd batch-2\nls   # protocol and reference submission (batch-2-design), problems, human and AI solutions, referee reports","problems":[{"group":"batch-2","description":"Batch 2 problem by Larry Guth. No passing proof yet.","metricUnit":"of 3","id":"b2-p04","name":"P4 · Metric geometry","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":true,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Reject · $951 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-A/submission-4A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission A: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $208 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-B/submission-4B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $10 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-C/submission-4C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $79 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-D/submission-4D-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Sam Payne, Jidong (Jayden) Wang. No passing proof yet.","metricUnit":"of 3","id":"b2-p08","name":"P8 · Matroids and tropical geometry","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":true,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Reject · $321 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-A/submission-8A-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission A: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Major revisions · $757 API","date":"2026-06-10","value":1,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-B/submission-8B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission B: \"Major revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $14 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-C/submission-8C-reviews.pdf"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $198 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-D/submission-8D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Srivatsav Kunnawalkam Elayavalli. No passing proof yet.","metricUnit":"of 3","id":"b2-p10","name":"P10 · Von Neumann algebras","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":true,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Major revisions · $673 API","date":"2026-06-10","value":1,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-A/submission-10A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission A: \"Major revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $589 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-B/submission-10B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Major revisions · $8 API","date":"2026-06-10","value":1,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-C/submission-10C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission C: \"Major revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $100 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-D/submission-10D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Sum of editorial grades over the Batch 2 problems: 3 essentially flawless, 2 minor revisions (passing), 1 major revisions, 0 reject or missing. One shot per problem, run by the editorial board.","metricUnit":"of 30","id":"b2-systems","name":"Batch 2 · all 10 problems","metricName":"total referee grade","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · 6/10 passed (2 flawless, 4 minor, 1 major, 2 reject, 1 missing) · $3,186 API, 22.9 h · gpt-5.5 pro, gpt-5.5, gemini-3.1-pro-preview, claude-opus-4-7","date":"2026-06-10","value":15,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119): System A, 9 graded solutions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · 5/10 passed (1 flawless, 4 minor, 1 major, 4 reject) · $4,799 API, 23.1 h · gpt-5.5 pro","date":"2026-06-10","value":12,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119): System B, 10 graded solutions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · 5/10 passed (2 flawless, 3 minor, 1 major, 4 reject) · $117 API, 5.8 h","date":"2026-06-10","value":13,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119): System C, 10 graded solutions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · 1/10 passed (1 minor, 9 reject) · $1,014 API, 7.8 h · gemini-3.1-pro-preview","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119): System D, 10 graded solutions","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Ted Slaman, Dariusz Kalociński. Passed by systems A, B, C.","metricUnit":"of 3","id":"b2-p01","name":"P1 · Computability theory","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $49 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-A/submission-1A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $189 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-B/submission-1B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Minor revisions · $13 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-C/submission-1C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission C: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $142 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-D/submission-1D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Richard Schwartz. Passed by systems A, B, C.","metricUnit":"of 3","id":"b2-p02","name":"P2 · Discrete geometry","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $86 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-A/submission-2A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $562 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-B/submission-2B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Minor revisions · $12 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-C/submission-2C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission C: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $85 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-D/submission-2D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Aleksa Milojević, Benny Sudakov. Passed by system A.","metricUnit":"of 3","id":"b2-p03","name":"P3 · Discrete probability","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $712 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-A/submission-3A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $637 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-B/submission-3B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $16 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-C/submission-3C-reviews.pdf"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $80 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-D/submission-3D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Oleg Butkovsky, Jonathan C. Mattingly, Lorenzo Zambotti. Passed by system A.","metricUnit":"of 3","id":"b2-p05","name":"P5 · Stochastic PDE","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Essentially flawless · $313 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-A/submission-5A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission A: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $642 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-B/submission-5B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $11 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-C/submission-5C-reviews.pdf"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $61 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-D/submission-5D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Joshua Greene, Duncan McCoy. Passed by systems B, C. System A: no solution.","metricUnit":"of 3","id":"b2-p06","name":"P6 · Lattice theory","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"UCLA Moonshot Harness","description":"System B · Essentially flawless · $218 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-06/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-06/submission-B/submission-6B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 6, Submission B: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Essentially flawless · $11 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-06/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-06/submission-C/submission-6C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 6, Submission C: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $72 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-06/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-06/submission-D/submission-6D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 6, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Sucharit Sarkar. Passed by systems A, B, C, D.","metricUnit":"of 3","id":"b2-p07","name":"P7 · Combinatorial topology","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Essentially flawless · $44 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-A/submission-7A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission A: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $480 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-B/submission-7B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Essentially flawless · $13 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-C/submission-7C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission C: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Minor revisions · $115 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-D/submission-7D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission D: \"Minor revisions\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Sylvie Corteel, John Lentfer. Passed by systems A, B, C.","metricUnit":"of 3","id":"b2-p09","name":"P9 · Algebraic combinatorics","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $36 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-A/submission-9A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $516 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-B/submission-9B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Minor revisions · $8 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-C/submission-9C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission C: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $81 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-D/submission-9D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"1stproof","name":"batch-2","branch":"main"},"participants":[{"name":"UCLA Moonshot Harness","url":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissions":11,"problems":["b2-p04","b2-p08","b2-p10","b2-systems","b2-p01","b2-p02","b2-p03","b2-p05","b2-p06","b2-p07","b2-p09"],"bestRank":1,"lastActive":"2026-06-10"},{"name":"OpenAI ChatGPT 5.5 Pro","url":"https://arxiv.org/abs/2606.18119","submissions":11,"problems":["b2-p04","b2-p08","b2-p10","b2-systems","b2-p01","b2-p02","b2-p03","b2-p05","b2-p06","b2-p07","b2-p09"],"bestRank":1,"lastActive":"2026-06-10"},{"name":"Princeton Momus","url":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissions":11,"problems":["b2-p04","b2-p08","b2-p10","b2-systems","b2-p01","b2-p02","b2-p03","b2-p05","b2-p06","b2-p07","b2-p09"],"bestRank":1,"lastActive":"2026-06-10"},{"name":"IMProofBench ProofCouncil","url":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissions":10,"problems":["b2-p04","b2-p08","b2-p10","b2-systems","b2-p01","b2-p02","b2-p03","b2-p05","b2-p07","b2-p09"],"bestRank":1,"lastActive":"2026-06-10"}],"stats":{"totalRecords":43,"uniqueParticipants":4,"firstSubmission":"2026-06-10","lastSubmission":"2026-06-10","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.136Z"},{"id":"sutro-problems","name":"Sutro Problems","tagline":"Optimize machine learning and matrix multiplication algorithms for energy efficiency on physical cost models.","description":"A benchmark suite for energy-efficient learning algorithms, featuring matrix multiplication, sparse parity, symmetry, and MNIST. Solutions are evaluated on Bill Dally's 2D physical data-movement cost model or real NVIDIA A100 GPUs. Run by Cybertron AI to explore alternatives to backpropagation under memory-wall bottlenecks.","url":"https://github.com/cybertronai/sutro-problems","host":"github","status":"active","domain":"hardware-efficiency","tags":["energy-efficiency","hardware-aware","matrix-multiplication","mnist","sparse-parity","compilers"],"images":["https://github.com/user-attachments/assets/dfab8831-876a-4cdd-bda4-79a7ef869152","https://github.com/user-attachments/assets/e4610f22-00e6-4d70-adee-1948f219cf51","https://github.com/user-attachments/assets/0d71bd80-7da8-4e0c-942d-7bd6b741a193"],"participation":{"howToSubmit":"Open a pull request against main adding your .ir and generator under submissions/ (for matmul/sparse-parity) or your source/generator under mnist/submissions/<name>/.","submissionFormat":"Straight-line program (IR) files, Python generators, and Markdown reports.","computeDetails":"Symbolic evaluation for matmul, symmetry, and sparse parity (no GPU needed). NVIDIA A100-80GB (via Modal) is used for the MNIST-A100 tracks.","deadline":"rolling","prizes":"recognition only","verification":"CI symbolic check for matmul/symmetry, deterministic simulation for sparse parity, and sandboxed Modal A100 runs for MNIST.","requirements":["Python 3","NumPy","PyTorch","CUDA"],"compute":"datacenter-gpu"},"links":[{"label":"Sutro Group Research","url":"https://docs.google.com/document/d/1B9867EN6Bg4ZVQK9vI_ZqykZ5HEtMAHJ7zBGGas4szQ/edit?tab=t.0#heading=h.j6rssh3enbtd"},{"label":"Bill Dally Model","url":"https://github.com/cybertronai/simplified-dally-model"}],"organizer":{"name":"Cybertron AI","url":"https://github.com/cybertronai","type":"company"},"quickstart":"import matmul\nir = matmul.generate_baseline_4x4()\ncost = matmul.score_4x4(ir)\nprint(f'Baseline 4x4 cost: {cost}')","problems":[{"group":"mnist-a100","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 3 requires error <= 2.70%.","metricUnit":"ms","id":"mnist-a100-diff3","name":"MNIST-a100 (Difficulty 3)","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"mnist-a100","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 4 requires error <= 2.30%.","metricUnit":"ms","id":"mnist-a100-diff4","name":"MNIST-a100 (Difficulty 4)","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"mnist-a100","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 5 requires error <= 1.90%.","metricUnit":"ms","id":"mnist-a100-diff5","name":"MNIST-a100 (Difficulty 5)","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"mnist-a100","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 1 requires error <= 5.40%.","metricUnit":"ms","id":"mnist-a100-diff1","name":"MNIST-a100 (Difficulty 1)","metricName":"time","metricDirection":"minimize","baseline":700.1,"isOpen":false,"records":[{"contributor":"yaroslavvb","description":"example.py baseline: MLP 60-1024-1024-10, 400 steps","date":"2026-09-25","value":700.1,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","isBaseline":true},{"contributor":"yaroslavvb","description":"MLP 60-1024-1024-10, 100 steps","date":"2026-09-26","value":191.3,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"yaroslavvb","description":"MLP 60-256-256-10, 200 steps in a CUDA graph ★ best","date":"2026-09-26","value":61.7,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","isCurrentBest":true,"agent":{"model":"Claude Opus 5.5","evidence":"Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"mnist-a100","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 2 requires error <= 3.40%.","metricUnit":"ms","id":"mnist-a100-diff2","name":"MNIST-a100 (Difficulty 2)","metricName":"time","metricDirection":"minimize","baseline":927.3,"isOpen":false,"records":[{"contributor":"yaroslavvb","description":"16 MLPs 60-1024-1024-10, 400 steps","date":"2026-09-26","value":927.3,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","isBaseline":true},{"contributor":"yaroslavvb","description":"4 MLPs 60-256-256-10, 800 steps in a CUDA graph ★ best","date":"2026-09-26","value":252.1,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","isCurrentBest":true,"agent":{"model":"Claude Opus 5.5","evidence":"Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"mnist","description":"MNIST-medium requires at least 97% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricUnit":"mJ","id":"mnist-medium-3","name":"MNIST-medium (3% error target)","metricName":"energy","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"mnist","description":"MNIST-medium requires at least 92% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricUnit":"mJ","id":"mnist-medium-8","name":"MNIST-medium (8% error target)","metricName":"energy","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"mnist","description":"MNIST-original requires at least 99% accuracy on the official 10,000-image test split of 28x28 pixels.","metricUnit":"mJ","id":"mnist-original","name":"MNIST-original (1% test error target)","metricName":"energy","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"mnist","description":"MNIST-small requires 67% test accuracy on 1,000 training / 1,000 test images of 3x3 pixels.","metricUnit":"mJ","id":"mnist-small-67","name":"MNIST-small (67% accuracy target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"H32 MLP","description":"H32 MLP","date":"2026-09-11","value":3300,"contributorKind":"method"},{"contributor":"NR-K8 Adam MLP","description":"NR-K8 Adam MLP","date":"2026-09-14","value":2600,"contributorKind":"method"},{"contributor":"Panel-cached H32 MLP","description":"Panel-cached H32 MLP (1,000/1,000)","date":"2026-09-15","value":3200,"contributorKind":"method"},{"contributor":"QDA","description":"QDA","date":"2026-09-15","value":0.59,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Claude Fable 5.1","evidence":"Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}}],"contested":true},{"group":"mnist","description":"MNIST-medium requires at least 98% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricUnit":"mJ","id":"mnist-medium-2","name":"MNIST-medium (2% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"512-filter CG pair","description":"512-filter CG pair","date":"2026-09-16","value":65000,"contributorKind":"method","isCurrentBest":true}],"contested":false},{"group":"mnist","description":"MNIST-medium requires at least 95% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricUnit":"mJ","id":"mnist-medium-5","name":"MNIST-medium (5% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"512-unit MLP","description":"512-unit MLP (96% target)","date":"2026-09-11","value":290000,"contributorKind":"method","agent":{"tool":"Codex","evidence":"Merge pull request #66 from cybertronai/codex/mnist-medium-submission","family":"openai","confidence":"medium","role":"author"}},{"contributor":"PCA-QDA","description":"PCA-QDA","date":"2026-09-15","value":174,"contributorKind":"method","isCurrentBest":true,"agent":{"tool":"Codex","evidence":"Merge pull request #85 from cybertronai/codex/pca-qda-energy-correction","family":"openai","confidence":"medium","role":"author"}}],"contested":true},{"group":"mnist","description":"MNIST-medium requires at least 88% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricUnit":"mJ","id":"mnist-medium-12","name":"MNIST-medium (12% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Reversible MLP","description":"Reversible MLP, smaller workspace (12% error target)","date":"2026-09-14","value":1000,"contributorKind":"method","isCurrentBest":true,"agent":{"tool":"Codex","evidence":"Merge pull request #74 from cybertronai/codex/rev88-20260914","family":"openai","confidence":"medium","role":"author"}}],"contested":false},{"group":"mnist","description":"Historical submissions for MNIST-small under the previous specification (600 training / 600 test images).","metricUnit":"mJ","id":"mnist-small-historical","name":"MNIST-small (Historical)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Panel-cached MLP","description":"Panel-cached MLP (600/600)","date":"2026-09-12","value":1800,"contributorKind":"method"},{"contributor":"32-unit MLP","description":"32-unit MLP","date":"2026-09-10","value":2000,"contributorKind":"method"},{"contributor":"1NN","description":"1NN","date":"2026-09-10","value":0.52,"contributorKind":"method","isCurrentBest":true,"agent":{"tool":"Codex","evidence":"Merge pull request #64 from cybertronai/codex/mnist-small-1nn-submission","family":"openai","confidence":"medium","role":"author"}}],"contested":true},{"group":"mnist","description":"Historical submissions for MNIST-medium under the previous specification (6,000 training / 6,000 test images).","metricUnit":"mJ","id":"mnist-medium-historical","name":"MNIST-medium (Historical)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Ordered ConvNets","description":"Ordered ConvNets (3% error target)","date":"2026-09-11","value":780000,"contributorKind":"method","agent":{"tool":"Codex","evidence":"Merge pull request #68 from cybertronai/codex/mnist-medium-convnet-submission","family":"openai","confidence":"medium","role":"author"}},{"contributor":"Three ConvNets","description":"Three ConvNets","date":"2026-09-10","value":1700000,"contributorKind":"method"},{"contributor":"512-unit MLP","description":"512-unit MLP","date":"2026-09-10","value":150000,"contributorKind":"method","isCurrentBest":true}],"contested":true},{"group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit) with a 20% accuracy target.","id":"sparse-parity-20","name":"Sparse Parity (20% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"b0nce","description":"generate_packed_static20() (packed static information set)","date":"2026-09-02","value":86753,"contributorKind":"person","contributorUrl":"https://github.com/b0nce","isCurrentBest":true},{"contributor":"jurajselep","description":"generate_packed_scan(1) (packed-column scan + SSA layout)","date":"2026-09-01","value":135348,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"npow","description":"generate_packed_sis(cap=2, seed=13, g2=8) (packed SIS, partial cap-2 walk, exhaustive information-set tuning)","date":"2026-08-31","value":151117,"contributorKind":"person","contributorUrl":"https://github.com/npow"},{"contributor":"zh4ngx","description":"optimize_layout(generate_sis_mask(1, 2)) (static-IS walk, staged layout)","date":"2026-08-30","value":1317480,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"yaroslavvb","description":"generate_scan(127) (Gray scan)","date":"2026-08-27","value":17331683,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"yaroslavvb","description":"generate_isd_mask(8) (ISD restarts)","date":"2026-08-26","value":12042480,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"}],"contested":true},{"group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit) with a 40% accuracy target.","id":"sparse-parity-40","name":"Sparse Parity (40% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"npow","description":"55 audited weight-2 states + compact flow + reverse route ★ best","date":"2026-09-01","value":141218,"contributorKind":"person","contributorUrl":"https://github.com/npow","isCurrentBest":true},{"contributor":"b0nce","description":"generate_packed_route40() (cap-2 Gray prefix)","date":"2026-09-02","value":147000,"contributorKind":"person","contributorUrl":"https://github.com/b0nce"},{"contributor":"jurajselep","description":"generate_packed_scan(2) (packed-column scan + SSA layout)","date":"2026-09-01","value":151943,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"npow","description":"generate(1, 2, seed=5) (bit-packed SIS walk, 3-phase layout, higher dev recovery)","date":"2026-08-31","value":163378,"contributorKind":"person","contributorUrl":"https://github.com/npow"},{"contributor":"zh4ngx","description":"optimize_layout(generate_sis_mask(1, 2)) (static-IS walk, staged layout)","date":"2026-08-30","value":1317480,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"zh4ngx","description":"generate_scan(0, walk=\"weight\", weight_cap=2) (weight-ordered scan)","date":"2026-08-28","value":17418235,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"yaroslavvb","description":"generate_scan(1023) (Gray scan)","date":"2026-08-27","value":18764343,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"yaroslavvb","description":"generate_scan(511) (Gray scan)","date":"2026-08-27","value":17945660,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"}],"contested":true},{"group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit) with a 60% accuracy target.","id":"sparse-parity-60","name":"Sparse Parity (60% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"npow","description":"19 audited weight-3 states + compact flow + reversed 2-opt walk ★ best","date":"2026-09-01","value":149665,"contributorKind":"person","contributorUrl":"https://github.com/npow","isCurrentBest":true},{"contributor":"b0nce","description":"generate_packed_route60() (cap-3 Gray prefix)","date":"2026-09-02","value":176331,"contributorKind":"person","contributorUrl":"https://github.com/b0nce"},{"contributor":"jurajselep","description":"generate_packed_scan(3) (packed-column scan + SSA layout)","date":"2026-09-01","value":200937,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"npow","description":"generate_packed_sis(cap=3, seed=13) (bit-packed SIS walk, full cap-3, tuned seed)","date":"2026-08-31","value":284049,"contributorKind":"person","contributorUrl":"https://github.com/npow"},{"contributor":"zh4ngx","description":"optimize_layout(generate_sis_mask(1, 3)) (static-IS walk, staged layout)","date":"2026-08-30","value":2137725,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"zh4ngx","description":"generate_scan(0, walk=\"weight\", weight_cap=3) (weight-ordered scan)","date":"2026-08-28","value":18509753,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"yaroslavvb","description":"generate_scan(6143) (Gray scan)","date":"2026-08-27","value":26951367,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"yaroslavvb","description":"generate_scan(4095) (Gray scan)","date":"2026-08-26","value":23676539,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"}],"contested":true},{"group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit) with an 80% accuracy target.","id":"sparse-parity-80","name":"Sparse Parity (80% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"npow","description":"269 audited weight-3 states + compact flow + reversed 2-opt walk ★ best","date":"2026-09-01","value":182744,"contributorKind":"person","contributorUrl":"https://github.com/npow","isCurrentBest":true},{"contributor":"b0nce","description":"generate_packed_route80() (cap-3 Gray prefix)","date":"2026-09-02","value":196139,"contributorKind":"person","contributorUrl":"https://github.com/b0nce"},{"contributor":"jurajselep","description":"generate_packed_scan(3) (packed-column scan + SSA layout)","date":"2026-09-01","value":200937,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"npow","description":"generate_staged(weight_cap=3) (septet-packed dynamic RREF + row-coordinate walk)","date":"2026-08-31","value":493193,"contributorKind":"person","contributorUrl":"https://github.com/npow"},{"contributor":"zh4ngx","description":"optimize_layout(generate_scan(0, walk=\"weight\", weight_cap=3)) (weight-ordered scan, staged layout)","date":"2026-08-30","value":5593997,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"yaroslavvb","description":"generate_scan(10239) (Gray scan)","date":"2026-08-27","value":33501030,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"yaroslavvb","description":"generate_scan(8191) (Gray scan)","date":"2026-08-27","value":30226172,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"}],"contested":true},{"group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit) with a 100% accuracy target.","id":"sparse-parity-100","name":"Sparse Parity (100% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"jurajselep","description":"generate_packed_scan(5) (packed RREF + specialized capture)","date":"2026-09-02","value":392666,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","isCurrentBest":true},{"contributor":"npow","description":"generate_staged() (septet-packed RREF + row-coordinate walk)","date":"2026-08-31","value":938331,"contributorKind":"person","contributorUrl":"https://github.com/npow"},{"contributor":"zh4ngx","description":"optimize_layout(generate_scan(0, walk=\"weight\", weight_cap=5)) (weight-ordered scan, staged layout)","date":"2026-08-30","value":12461610,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"yaroslavvb","description":"generate_scan(16383) (Gray scan, full walk)","date":"2026-08-26","value":43325468,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"}],"contested":true},{"group":"symmetry","description":"Classify a 6-bit pattern as a palindrome (1) or not (0) at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit).","id":"symmetry-6bit","name":"Symmetry (6-bit, 100% target)","metricName":"cost","metricDirection":"minimize","baseline":20,"isOpen":true,"records":[{"contributor":"baseline","description":"generate_baseline_six (3× cmp eq + 2× and)","date":"2026-05-11","value":20,"contributorKind":"method","isBaseline":true,"isCurrentBest":true}],"contested":false},{"group":"symmetry","description":"Classify an 8-bit pattern as a palindrome (1) or not (0) at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit).","id":"symmetry-8bit","name":"Symmetry (8-bit, 100% target)","metricName":"cost","metricDirection":"minimize","baseline":29,"isOpen":true,"records":[{"contributor":"baseline","description":"generate_baseline_eight (4× cmp eq + 3× and)","date":"2026-05-11","value":29,"contributorKind":"method","isBaseline":true,"isCurrentBest":true}],"contested":false},{"group":"matmul","description":"Ordinary 4x4 matrix multiplication optimized for energy/read-cost on Bill Dally's physical cost model.","id":"matmul-4x4","name":"4x4 Matrix Multiplication","metricName":"cost","metricDirection":"minimize","baseline":1316,"isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_baseline_4x4 (naive)","date":"2026-04-29","value":1316,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","isBaseline":true},{"contributor":"sjbaebae","description":"generate_outer_product_4x4 (size-1 sA)","date":"2026-04-30","value":800,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae"},{"contributor":"jurajselep","description":"row-wise lifetime fusion + dead-A output reuse + exact tier allocation","date":"2026-09-01","value":689,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"npow","description":"HKKK + one deferred output add + exact fixed-schedule allocation","date":"2026-09-01","value":683,"contributorKind":"person","contributorUrl":"https://github.com/npow"},{"contributor":"jurajselep","description":"hybrid row-0 schedule + JIT A staging + interleaved final adds","date":"2026-09-01","value":681,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"jurajselep","description":"selective A/B lifetime splits + dependency-safe scheduling + exact lifetime coloring","date":"2026-09-04","value":675,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","isCurrentBest":true}],"contested":true},{"group":"matmul","description":"Ordinary 16x16 matrix multiplication optimized for energy/read-cost on Bill Dally's physical cost model.","id":"matmul-16x16","name":"16x16 Matrix Multiplication","metricName":"cost","metricDirection":"minimize","baseline":340704,"isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_baseline_16x16 (naive)","date":"2026-04-29","value":340704,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","isBaseline":true},{"contributor":"yaroslavvb","description":"generate_recursive_16x16 (1×1-leaf D&C, Z-order)","date":"2026-05-08","value":237456,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"yaroslavvb","description":"generate_tiled_16x16 (4×4 tiles)","date":"2026-04-29","value":133783,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"SethTS","description":"generate_tiled_16x16_opt1 (tmp@1)","date":"2026-04-30","value":110743,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","agent":{"model":"Claude Sonnet 4.6","evidence":"Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"sjbaebae","description":"generate_hierarchical_16x16 (asym. reload)","date":"2026-04-30","value":80217,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae"},{"contributor":"adotzh","description":"sA-cache + sB scratchpad (rank2)","date":"2026-05-01","value":73602,"contributorKind":"person","contributorUrl":"https://github.com/adotzh"},{"contributor":"sjbaebae","description":"+ redirect last-mul to addr 1","date":"2026-05-01","value":72642,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae"},{"contributor":"sjbaebae","description":"+ last-super-block outputs in sC","date":"2026-05-01","value":71724,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae"},{"contributor":"sjbaebae","description":"+ dead-input output reuse + B packing","date":"2026-05-01","value":70053,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae"},{"contributor":"yaroslavvb","description":"C↔A address aliasing + final-add fusion","date":"2026-05-06","value":69697,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb"},{"contributor":"zh4ngx","description":"+ column-major order + fused final copy-out","date":"2026-05-05","value":68452,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx"},{"contributor":"cosminscn","description":"+ liveness order + output-read-aware packing + five-output scratch tail","date":"2026-05-13","value":68390,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin","family":"openai","confidence":"medium","role":"author"}},{"contributor":"cosminscn","description":"+ live-B evacuation + output deferral + A staging + value-lifetime coloring","date":"2026-05-13","value":67834,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"model":"Codex","evidence":"Author: Codex and Cosmin","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"+ live-B evacuation + output deferral + tiny A-staging mask + staged-reload endpoint lift + value-lifetime coloring","date":"2026-05-14","value":67821,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"model":"Codex","evidence":"Author: Codex and Cosmin","family":"openai","confidence":"high","role":"author"}},{"contributor":"sjbaebae","description":"weighted-lifetime pressure search + copy elimination","date":"2026-05-08","value":66707,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","agent":{"tool":"Codex","evidence":"[codex] add weighted lifetime matmul submission","family":"openai","confidence":"medium","role":"author"}},{"contributor":"cosminscn","description":"macro B-staging + row-7 later-panel prestaging from addr 1","date":"2026-05-25","value":66633,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"model":"Codex","evidence":"Author: Codex and Cosmin","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"late B-block cheap capture from addr 1 + value-lifetime coloring","date":"2026-05-25","value":66524,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"model":"Codex","evidence":"Author: Codex and Cosmin","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"late copy-schedule motif bundle + value-lifetime coloring","date":"2026-05-26","value":66400,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"model":"Codex","evidence":"Author: Codex and Cosmin","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"Claude-assisted simulated annealing over a leaderboard physical-address IR","date":"2026-05-28","value":66300,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"model":"Claude","evidence":"Claude-assisted simulated annealing over a leaderboard physical-address IR","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"sigkillme0","description":"dependency-safe rescheduling + exact tier allocation","date":"2026-08-29","value":66199,"contributorKind":"person","contributorUrl":"https://github.com/sigkillme0"},{"contributor":"sigkillme0","description":"exact LP-optimal address assignment (provably optimal for this operation order)","date":"2026-08-30","value":66178,"contributorKind":"person","contributorUrl":"https://github.com/sigkillme0"},{"contributor":"jurajselep","description":"newest-first snake passes + JIT A staging + local exact tier allocation","date":"2026-09-04","value":65084,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"cosminscn","description":"asymmetric panel schedule + persistent B captures + dependency-safe order search","date":"2026-09-04","value":64431,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","agent":{"tool":"Codex","evidence":"Merge pull request #59 from cosminscn/codex/matmul-record-64431-clean","family":"openai","confidence":"medium","role":"author"}},{"contributor":"SecurityQQ","description":"temporary input captures + cheapest surviving replica reads + redundant-copy deletion","date":"2026-09-07","value":64074,"contributorKind":"person","contributorUrl":"https://github.com/SecurityQQ"},{"contributor":"SecurityQQ","description":"6+10 asymmetric panels + dependency-safe scheduling + input captures and address allocation (frozen-artifact verifier)","date":"2026-09-07","value":63819,"contributorKind":"person","contributorUrl":"https://github.com/SecurityQQ"},{"contributor":"jurajselep","description":"input copy-chain and neutral-plan optimization + whole-program allocation with an exact rational certificate","date":"2026-09-15","value":63639,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"jurajselep","description":"exact block reordering + joint reduction-tree and storage allocation","date":"2026-09-15","value":63354,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep"},{"contributor":"jurajselep","description":"joint reduction-tree/storage repair + redundant-copy elimination ★ best","date":"2026-09-18","value":63350,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","isCurrentBest":true}],"contested":true}],"agentStats":[{"family":"openai","records":13,"currentBests":2,"models":["Codex"],"contributors":["512-unit MLP","PCA-QDA","Reversible MLP","1NN","Ordered ConvNets","cosminscn","sjbaebae"],"uncontestedBests":1},{"family":"anthropic","records":5,"currentBests":1,"models":["Claude Opus 5.5","Claude Fable 5.1","Claude Sonnet 4.6","Claude"],"contributors":["yaroslavvb","QDA","SethTS","cosminscn"],"uncontestedBests":2}],"repo":{"owner":"cybertronai","name":"sutro-problems","branch":"main"},"participants":[{"name":"jurajselep","url":"https://github.com/jurajselep","submissions":12,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-4x4","matmul-16x16"],"lastActive":"2026-09-18","bestRank":1},{"name":"npow","url":"https://github.com/npow","submissions":9,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-4x4"],"lastActive":"2026-09-01","bestRank":1},{"name":"b0nce","url":"https://github.com/b0nce","submissions":4,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80"],"bestRank":1,"lastActive":"2026-09-02"},{"name":"yaroslavvb","url":"https://github.com/yaroslavvb","submissions":19,"problems":["mnist-a100-diff1","mnist-a100-diff2","sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-4x4","matmul-16x16"],"lastActive":"2026-09-26"},{"name":"zh4ngx","url":"https://github.com/zh4ngx","submissions":8,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-16x16"],"lastActive":"2026-08-30"},{"name":"cosminscn","url":"https://github.com/cosminscn","submissions":8,"problems":["matmul-16x16"],"lastActive":"2026-09-04"},{"name":"sjbaebae","url":"https://github.com/sjbaebae","submissions":6,"problems":["matmul-4x4","matmul-16x16"],"lastActive":"2026-05-08"},{"name":"sigkillme0","url":"https://github.com/sigkillme0","submissions":2,"problems":["matmul-16x16"],"lastActive":"2026-08-30"},{"name":"SecurityQQ","url":"https://github.com/SecurityQQ","submissions":2,"problems":["matmul-16x16"],"lastActive":"2026-09-07"},{"name":"SethTS","url":"https://github.com/SethTS","submissions":1,"problems":["matmul-16x16"],"lastActive":"2026-04-30"},{"name":"adotzh","url":"https://github.com/adotzh","submissions":1,"problems":["matmul-16x16"],"lastActive":"2026-05-01"}],"stats":{"totalRecords":88,"uniqueParticipants":11,"firstSubmission":"2026-04-29","lastSubmission":"2026-09-26","recordsLast30d":14,"recordsLast90d":65},"lastUpdated":"2026-10-13T12:55:04.138Z"},{"id":"lean-eval","name":"LeanEval","tagline":"Prove open formalization problems in Lean 4; models and submitters are ranked by problems solved.","description":"LeanEval is the Lean FRO's comparator-based benchmark for formal mathematics. Trusted theorem statements live in leanprover/lean-eval; a problem counts as solved when the comparator accepts a submitted proof. Results are sticky per (user, model, problem) and published at lean-lang.org/eval. 304 visible problems, 75 models, 60 submitters, 39 active problems still unsolved.","url":"https://lean-lang.org/eval/","host":"website","status":"active","domain":"formal-methods","tags":["lean","theorem-proving","formal-methods","mathlib","benchmark"],"images":[],"participation":{"howToSubmit":"Prove a problem's theorem in your own GitHub repository and submit the ref at lean-lang.org/eval/submit; the comparator verifies the proof and the result is appended to results/<login>.json in leanprover/lean-eval-submissions.","submissionFormat":"GitHub repo + ref containing the completed Lean proof, plus the declared model label used to produce it.","computeDetails":"Scoring is a comparator check of the proof; whatever compute the prover needs is on the submitter's side.","deadline":"rolling","prizes":"recognition only","eligibility":"Open to everyone. Successes are sticky: an accepted (user, model, problem) tuple is never removed.","cost":"free","verification":"Comparator acceptance in CI, replay by maintainers, public audit log per user.","requirements":["Lean 4","Mathlib","GitHub account"],"compute":"none"},"links":[{"label":"Leaderboard","url":"https://lean-lang.org/eval/"},{"label":"Submit a solution","url":"https://lean-lang.org/eval/submit/"},{"label":"Benchmark problems (leanprover/lean-eval)","url":"https://github.com/leanprover/lean-eval"},{"label":"Results store (lean-eval-submissions)","url":"https://github.com/leanprover/lean-eval-submissions"}],"organizer":{"name":"Lean FRO","url":"https://lean-lang.org/","type":"foundation"},"venue":{"location":"GitHub / lean-lang.org","type":"online"},"quickstart":"git clone https://github.com/leanprover/lean-eval && cd lean-eval\nlake exe cache get && lake build\nlake exe lean-eval --help   # pick a problem under generated/, replace sorry, submit at lean-lang.org/eval/submit","problems":[{"description":"Unsolved. Group: formalization-evaluation. Source: M. R. Bridson, D. B. McReynolds, A. W. Reid, and R. Spitler, `Absolute profinite rigidity and hyperbolic geometry`, Annals of Math, 192 (3) 2020. Statement take Problem by Thomas Browning.","id":"open-annals_absolute_profinite_rigidity","name":"Absolute profinite rigidity and hyperbolic geometry","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: W. Sawin and M. Shusterman, `On the Chowla and twin primes conjectures over 𝔽_q[T]`, Annals of Math, 196 (2) 2022. Statement taken from https://github.com/Impe Problem by Thomas Browning.","id":"open-annals_chowla_and_twin_prime_over_fq_t","name":"On the Chowla and twin primes conjectures over 𝔽_q[T]","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: I. Petrow and M. P. Young, `The Weyl bound for Dirichlet L-functions of cube-free conductor`, Annals of Math, 192 (2) 2020. Statement taken from https://github. Problem by Thomas Browning.","id":"open-annals_dirichlet_weyl_bound","name":"The Weyl bound for Dirichlet L-functions of cube-free conductor","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. Bennett and S. Siksek, `A conjecture of Erdős, supersingular primes and short character sums`, Annals of Math, 191 (2) 2020. Statement taken from https://git Problem by Justus Springer.","id":"open-annals_erdos_supersingular_primes","name":"A conjecture of Erdős, supersingular primes and short character sums","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. Cabanes and B. Späth, `The McKay Conjecture on character degrees`, Annals of Math, 203 (3) 2026. Statement taken from https://github.com/ImperialCollegeLondo Problem by Thomas Browning.","id":"open-annals_mckay_conjecture","name":"The McKay Conjecture on character degrees","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: D. Cristofaro-Gardiner, V. Humilière, and S. Seyfaddini, `Proof of the simplicity conjecture`, Annals of Math, 199 (1) 2024. Statement taken from https://github Problem by Thomas Browning.","id":"open-annals_simplicity_conjecture","name":"Proof of the simplicity conjecture","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: T. C. Burness, R. M. Guralnick, and S. Harper, `The spread of a finite group`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/ImperialCol Problem by Katerina Hristova.","id":"open-annals_spread_of_a_finite_group","name":"The spread of a finite group","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: J. Fernández de Bobadilla and T. Pełka, `Symplectic monodromy at radius zero and equimultiplicity of μ-constant families`, Annals of Math, 200 (1) 2024. Stateme Problem by Justus Springer.","id":"open-annals_symplectic_monodromy","name":"Symplectic monodromy at radius zero and equimultiplicity of μ-constant families","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: V. Dimitrov, Z. Gao, and P. Habegger, `Uniformity in Mordell–Lang for curves`, Annals of Math, 194 (1) 2021. Statement taken from https://github.com/ImperialCol Problem by Thomas Browning, Christian Merten.","id":"open-annals_uniform_mordell_lang","name":"Uniformity in Mordell–Lang for curves","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: F. Quinn, *Ends of maps III: dimensions 4 and 5*, J. Differential Geom. 17 (1982), building on M. H. Freedman, *The topology of four-dimensional manifolds*, J.  Problem by Kim Morrison.","id":"open-annulus_theorem_dim_four","name":"The Annulus Theorem in dimension 4 (Quinn)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: R. C. Kirby, *Stable homeomorphisms and the annulus conjecture*, Ann. of Math. 89 (1969). Problem by Kim Morrison.","id":"open-annulus_theorem_high_dim","name":"The Annulus Theorem in dimension ≥ 5 (Kirby)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: R. Budney and D. Gabai, 'Knotted 3-balls in S⁴', Corollary 8.6, arXiv:1912.09029v3 (2021), https://arxiv.org/abs/1912.09029. Problem by Vasily Ilin.","id":"open-budney_gabai_knotted_three_spheres","name":"Budney--Gabai knotted three-spheres in S¹ × S³","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: J. Cerf, Sur les diffeomorphismes de la sphere de dimension trois, Lecture Notes in Mathematics 53, Springer (1968). Problem by Kim Morrison.","id":"open-cerf_gamma_four","name":"Cerf's theorem: every self-diffeomorphism of S3 is smoothly isotopic to a linear isometry","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: D. Gorenstein, R. Lyons and R. Solomon, The Classification of the Finite Simple Groups (AMS Mathematical Surveys and Monographs 40, 1994–); M. Aschbacher, R. Ly Problem by Kim Morrison.","id":"open-classification_finite_simple_groups","name":"The classification of finite simple groups","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: https://arxiv.org/abs/1808.02923 Problem by Kim Morrison.","id":"open-conway_knot_not_smoothly_slice","name":"The Conway knot is not smoothly slice","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Freedman, *The topology of four-dimensional manifolds*, J. Diff. Geom. 17 (1982). See also Freedman-Quinn, *Topology of 4-Manifolds*, Princeton 1990. Problem by Kim Morrison.","id":"open-conway_knot_topologically_slice","name":"The Conway knot is topologically slice","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: https://github.com/dagurtomas/LeanCondensed (LeanCondensed/Projects/DerivedSolidCWHomology.lean); D. Clausen and P. Scholze, lectures on analytic stacks and lig Problem by Dagur Asgeirsson.","id":"open-derived_solidification_free_CW_homology","name":"Derived solidification of free CW complexes (light condensed mathematics)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Casson, 1980s (unpublished); Akbulut-Matveyev, *A convex decomposition theorem for 4-manifolds*, IMRN 1998. See also Hedden-Kirk-Livingston, *Non-slice linear c Problem by Kim Morrison.","id":"open-exists_topologically_slice_not_smoothly_slice","name":"Existence of a topologically slice, not smoothly slice knot","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","id":"open-faltings","name":"Faltings' theorem (Mordell conjecture)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: https://en.wikipedia.org/wiki/Fermat%27s_Last_Theorem Problem by Xuanji Li.","id":"open-fermat_last_theorem","name":"Fermat's Last Theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Folklore via CFSG; classical work of Mathieu, Jordan; modern accounts in P. Cameron, Permutation Groups (1999). Problem by Kim Morrison.","id":"open-five_transitive_card_classification","name":"Possible orders of 5-transitive finite permutation groups","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Problem by Oliver Nash.","id":"open-four_manifold_not_smooth","name":"Freedman's non-smoothability theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Kepler conjecture 1611 (J. Kepler, *Strena seu de Nive Sexangula*); proof T. Hales, 'A proof of the Kepler conjecture', Ann. of Math. (2) 162 (2005) 1065–1185;  Problem by Kim Morrison.","id":"open-kepler_conjecture","name":"Kepler conjecture (optimal sphere packing in ℝ³)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: János Kollár, Max Lieblich, Martin Olsson, and Will Sawin. The Zariski topology, linear systems, and algebraic varieties. Problem by Junyan Xu.","id":"open-kollar_lieblich_olsson_sawin","name":"Topological reconstruction theorems for varieties","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: C. Manolescu, *Pin(2)-equivariant Seiberg–Witten Floer homology and the triangulation conjecture*, J. Amer. Math. Soc. 29 (2016), via the Galewski–Stern and Mat Problem by Kim Morrison.","id":"open-manolescu_triangulation_disproof","name":"Manolescu's disproof of the triangulation conjecture","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: B. Mazur, *Modular curves and the Eisenstein ideal*, Publ. IHÉS 47 (1977). Knill, *Some fundamental theorems in mathematics*, §139. Problem by Kim Morrison.","id":"open-mazur_torsion","name":"Mazur's torsion theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Nikolay Nikolov and Dan Segal, *On finitely generated profinite groups, I: strong completeness and uniform bounds*, Annals of Mathematics 165 (2007), no. 1, 171 Problem by Adam Topaz.","id":"open-nikolov_segal","name":"Nikolov–Segal strong completeness theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Martin W. Liebeck, E. A. O'Brien, Aner Shalev, and Pham Huu Tiep, *The Ore conjecture*, Journal of the European Mathematical Society 12 (2010), no. 4, 939–1008, Problem by Adam Topaz.","id":"open-ore_conjecture","name":"The Ore conjecture: every element of a finite nonabelian simple group is a commutator","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, three arXiv preprints 2002-2003 (math/0211159, math/0303109, math/0307245). Recorded as a `proof_wanted` in Mathlib/Geometry/Manifold/PoincareConje Problem by Kim Morrison.","id":"open-poincare_3d_smooth","name":"3D smooth Poincaré conjecture (Perelman)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, The entropy formula for the Ricci flow and its geometric applications (arXiv:math/0211159, 2002); Ricci flow with surgery on three-manifolds (arXiv Problem by Kim Morrison.","id":"open-poincare_3d_topological","name":"3D topological Poincaré conjecture (Perelman)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. H. Freedman, The topology of four-dimensional manifolds, J. Differential Geom. 17 (1982), 357-453. Specialization of `proof_wanted ContinuousMap.HomotopyEqui Problem by Kim Morrison.","id":"open-poincare_4d_topological","name":"4D topological Poincaré conjecture (Freedman)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: S. Ramanujan, 'On certain arithmetical functions', Trans. Cambridge Philos. Soc. 22 (1916) 159–184. H. Petersson, 'Theorie der automorphen Formen beliebiger ree Problem by Seewoo Lee.","id":"open-ramanujan_petersson","name":"Ramanujan–Petersson conjecture for the τ-function (Deligne's theorem)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: O. Schreier, Über die Erweiterung von Gruppen II, Abh. Math. Sem. Univ. Hamburg 4 (1926); CFSG, completed c. 2004. Problem by Kim Morrison.","id":"open-schreier_conjecture","name":"Schreier's conjecture: outer automorphism group of a finite simple group is solvable","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: A. Hatcher, A proof of the Smale conjecture, Diff(S3) = O(4), Ann. of Math. 117 (1983). Problem by Kim Morrison.","id":"open-smale_conjecture","name":"Smale conjecture (Hatcher) in relative parameterized form","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. Bhargava, J. Hanke, Universal quadratic forms and the 290-theorem, preprint (2011). See also https://en.wikipedia.org/wiki/15_and_290_theorems Problem by Bolton Bailey/Project Numina.","id":"open-two_ninety_theorem","name":"The 290 theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: T. Watanabe, *Some exotic nontrivial elements of the rational homotopy groups of Diff(S⁴)*, arXiv:1812.02448 (2018). Pairs with the positive S³ statement of Hat Problem by Kim Morrison.","id":"open-watanabe_four_dim_smale_disproof","name":"Watanabe's disproof of the 4-dimensional Smale conjecture","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: H. A. Helfgott, The ternary Goldbach conjecture is true, arXiv:1312.7748 (2013), https://arxiv.org/abs/1312.7748; H. A. Helfgott and D. J. Platt, Numerical Veri Problem by Vasily Ilin.","id":"open-weak_goldbach","name":"Weak Goldbach theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: P. Deligne, 'La conjecture de Weil. I', Publications Mathématiques de l'Institut des Hautes Études Scientifiques, Volume 43, pages 273–307 (1974), https://www.n Problem by Junyan Xu.","id":"open-weil_conjectures","name":"Weil conjectures in terms of point counts","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: C.H. Taubes, 'The Seiberg–Witten equations and the Weinstein conjecture', Geom. Topol. 11 (2007) 2117–2202; arXiv:math/0611007. Conjecture: A. Weinstein, 'On th Problem by Kim Morrison.","id":"open-weinstein_conjecture_dim3","name":"Weinstein conjecture in dimension three (Taubes 2007)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Official leaderboard: each row is a model (or harness + model) ranked by the number of distinct benchmark problems it has solved; 304 problems in the catalog, 130 active.","id":"models","name":"Models — problems solved","metricName":"problems solved","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Axiom Prover (Axiom Math)","description":"1 submitter: Vilin97 (261)","date":"2026-10-13","value":261,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":true,"agent":{"model":"Axiom Prover (Axiom Math)","evidence":"lean-lang.org/eval leaderboard.json: \"Axiom Prover (Axiom Math)\" solved 261","family":"axiom","confidence":"high","role":"subject"}},{"contributor":"Humanifa + GPT 5.6 sol","description":"4 submitters: ZhengyangZhang06 (215), hongzhoulin89 (34), juihuichung (6), antoinegg1 (5)","date":"2026-10-13","value":256,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"lean-lang.org/eval leaderboard.json: \"Humanifa + GPT 5.6 sol\" solved 256","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Tau (caj.al)","description":"1 submitter: lukerj00 (200)","date":"2026-09-07","value":200,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Tau (caj.al)","evidence":"lean-lang.org/eval leaderboard.json: \"Tau (caj.al)\" solved 200","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Aristotle (Harmonic)","description":"7 submitters: LorenzoLuccioli (105), parabamoghv (42), JohnEdwardJennings (7), sqrt-of-2 (7), kim-em (6), adrianmartir (3), Parcly-Taxel (1)","date":"2026-08-20","value":162,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Aristotle (Harmonic)","evidence":"lean-lang.org/eval leaderboard.json: \"Aristotle (Harmonic)\" solved 162","family":"harmonic","confidence":"high","role":"subject"}},{"contributor":"Seed Prover (ByteDance)","description":"2 submitters: GanjinZero (135), hanwenzhu (21)","date":"2026-08-13","value":156,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Seed Prover (ByteDance)","evidence":"lean-lang.org/eval leaderboard.json: \"Seed Prover (ByteDance)\" solved 156","family":"bytedance","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6","description":"2 submitters: Morgan-Griffiths (111), eohjelle (3)","date":"2026-07-21","value":112,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6\" solved 112","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Stealth Model","description":"1 submitter: rishistyping (104)","date":"2026-08-20","value":104,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Stealth Model","evidence":"lean-lang.org/eval leaderboard.json: \"Stealth Model\" solved 104","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"NEAR AI w/ DeepSeek V4","description":"1 submitter: skidanovalex (85)","date":"2026-10-13","value":85,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"DeepSeek V4","tool":"NEAR AI","evidence":"lean-lang.org/eval leaderboard.json: \"NEAR AI w/ DeepSeek V4\" solved 85","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Vasily-24-hour-gpt-5.6-speedrun","description":"1 submitter: Vilin97 (79)","date":"2026-08-17","value":79,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Vasily-24-hour-gpt-5.6-speedrun","evidence":"lean-lang.org/eval leaderboard.json: \"Vasily-24-hour-gpt-5.6-speedrun\" solved 79","family":"openai","confidence":"high","role":"subject"}},{"contributor":"max gp (GPT-6)","description":"1 submitter: ma4ypic4y (73)","date":"2026-10-13","value":73,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"max gp (GPT-6)","evidence":"lean-lang.org/eval leaderboard.json: \"max gp (GPT-6)\" solved 73","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Aleph Prover(logicalintelligence.com)","description":"2 submitters: mayorov-m-a (52), antpavzhi (4)","date":"2026-06-05","value":56,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Aleph Prover(logicalintelligence.com)","evidence":"lean-lang.org/eval leaderboard.json: \"Aleph Prover(logicalintelligence.com)\" solved 56","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Midas","description":"1 submitter: sarjinius (39)","date":"2026-10-01","value":39,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Midas","evidence":"lean-lang.org/eval leaderboard.json: \"Midas\" solved 39","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"github.com/mlgraham (various models)","description":"1 submitter: mlgraham (31)","date":"2026-08-29","value":31,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"github.com/mlgraham (various models)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham (various models)\" solved 31","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"MerLean-Prover","description":"1 submitter: doxtor6 (30)","date":"2026-06-25","value":30,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"MerLean-Prover","evidence":"lean-lang.org/eval leaderboard.json: \"MerLean-Prover\" solved 30","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","description":"1 submitter: daouid (21)","date":"2026-05-22","value":21,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","evidence":"lean-lang.org/eval leaderboard.json: \"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)\" solved 21","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"savarin-6-hour-gpt-5.6-speedrun","description":"1 submitter: savarin (21)","date":"2026-08-17","value":21,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"savarin-6-hour-gpt-5.6-speedrun","evidence":"lean-lang.org/eval leaderboard.json: \"savarin-6-hour-gpt-5.6-speedrun\" solved 21","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7 (1M context)","description":"2 submitters: rkirov (15), jzuiddam (2)","date":"2026-05-24","value":17,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 4.7 (1M context)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7 (1M context)\" solved 17","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.5","description":"3 submitters: Morgan-Griffiths (10), sqrt-of-2 (9), A-M-Berns (4)","date":"2026-06-08","value":16,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.5","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.5\" solved 16","family":"openai","confidence":"high","role":"subject"}},{"contributor":"github.com/mlgraham (Claude Fable 5)","description":"1 submitter: mlgraham (16)","date":"2026-08-28","value":16,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"github.com/mlgraham (Claude Fable 5)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham (Claude Fable 5)\" solved 16","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"EVO (deepthought.com.au)","description":"1 submitter: test1-deepthought (8)","date":"2026-06-23","value":8,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"EVO (deepthought.com.au)","evidence":"lean-lang.org/eval leaderboard.json: \"EVO (deepthought.com.au)\" solved 8","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Public accepted source + Codex packaging","description":"1 submitter: rishistyping (7)","date":"2026-06-19","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Codex packaging","tool":"Public accepted source","evidence":"lean-lang.org/eval leaderboard.json: \"Public accepted source + Codex packaging\" solved 7","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5","description":"1 submitter: ZhengyangZhang06 (7)","date":"2026-08-21","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 5","tool":"Humanize2 flame chase: GPT-5.6-sol","evidence":"lean-lang.org/eval leaderboard.json: \"Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5\" solved 7","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Sol","description":"2 submitters: Vilin97 (5), n-yamaguchi-0729 (2)","date":"2026-09-08","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 Sol\" solved 7","family":"openai","confidence":"high","role":"subject"}},{"contributor":"trureturing","description":"1 submitter: AlyciaBHZ (7)","date":"2026-10-05","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"trureturing","evidence":"lean-lang.org/eval leaderboard.json: \"trureturing\" solved 7","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Opus-5","description":"1 submitter: Vilin97 (6)","date":"2026-08-13","value":6,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Opus-5","evidence":"lean-lang.org/eval leaderboard.json: \"Opus-5\" solved 6","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Fable 5","description":"2 submitters: rkirov (4), will1491 (1)","date":"2026-07-28","value":5,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5\" solved 5","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Forge","description":"1 submitter: xiaoyangliu-sjtu (5)","date":"2026-09-15","value":5,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Forge","evidence":"lean-lang.org/eval leaderboard.json: \"Forge\" solved 5","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Ivan Bogatyy","description":"1 submitter: bogatyy (4)","date":"2026-09-19","value":4,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Ivan Bogatyy","evidence":"lean-lang.org/eval leaderboard.json: \"Ivan Bogatyy\" solved 4","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4 Flash","description":"1 submitter: rwst (3)","date":"2026-07-12","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"DeepSeek V4 Flash","evidence":"lean-lang.org/eval leaderboard.json: \"DeepSeek V4 Flash\" solved 3","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Grok 4.5","description":"1 submitter: rwst (3)","date":"2026-07-23","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Grok 4.5","evidence":"lean-lang.org/eval leaderboard.json: \"Grok 4.5\" solved 3","family":"xai","confidence":"high","role":"subject"}},{"contributor":"deepseek-v4 and gpt-5.x","description":"1 submitter: ivwumupy (3)","date":"2026-07-27","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"deepseek-v4 and gpt-5.x","evidence":"lean-lang.org/eval leaderboard.json: \"deepseek-v4 and gpt-5.x\" solved 3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"mostly ChatGPT 5.6 Sol","description":"1 submitter: mccorvie (3)","date":"2026-08-03","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"mostly ChatGPT 5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"mostly ChatGPT 5.6 Sol\" solved 3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5.5 Codex","description":"1 submitter: A-M-Berns (2)","date":"2026-05-07","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.5 Codex\" solved 2","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Pro","description":"1 submitter: sqrt-of-2 (2)","date":"2026-05-10","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Gemini 3.1 Pro","evidence":"lean-lang.org/eval leaderboard.json: \"Gemini 3.1 Pro\" solved 2","family":"google","confidence":"high","role":"subject"}},{"contributor":"[submission] aegis-of-the-unit-circle-logos","description":"1 submitter: rishistyping (2)","date":"2026-05-12","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"[submission] aegis-of-the-unit-circle-logos","evidence":"lean-lang.org/eval leaderboard.json: \"[submission] aegis-of-the-unit-circle-logos\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"EVO","description":"2 submitters: machinelearning2014 (2), test1-deepthought (2)","date":"2026-05-30","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"EVO","evidence":"lean-lang.org/eval leaderboard.json: \"EVO\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.8 (1M context)","description":"1 submitter: rkirov (2)","date":"2026-07-01","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 4.8 (1M context)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.8 (1M context)\" solved 2","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Kimi K2.7","description":"1 submitter: rwst (2)","date":"2026-07-10","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Kimi K2.7","evidence":"lean-lang.org/eval leaderboard.json: \"Kimi K2.7\" solved 2","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 and Fable 5 (human-in-the-loop)","description":"1 submitter: KitaKen1 (2)","date":"2026-08-06","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 and Fable 5 (human-in-the-loop)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 and Fable 5 (human-in-the-loop)\" solved 2","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Humanize RLCR: GPT-5.6-sol (max) worker + reviewer","description":"1 submitter: ZhengyangZhang06 (2)","date":"2026-08-22","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"reviewer","tool":"Humanize RLCR: GPT-5.6-sol (max) worker","evidence":"lean-lang.org/eval leaderboard.json: \"Humanize RLCR: GPT-5.6-sol (max) worker + reviewer\" solved 2","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Forall (Astrio)","description":"1 submitter: nolanlwin (2)","date":"2026-09-07","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Forall (Astrio)","evidence":"lean-lang.org/eval leaderboard.json: \"Forall (Astrio)\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"trureturning","description":"1 submitter: AlyciaBHZ (2)","date":"2026-10-05","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"trureturning","evidence":"lean-lang.org/eval leaderboard.json: \"trureturning\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7","description":"1 submitter: rkirov (1)","date":"2026-04-30","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 4.7","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)","description":"1 submitter: yawara (1)","date":"2026-05-29","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.5 (human-in-the-loop)","tool":"Claude Opus 4.7","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5 Codex + Aristotle","description":"1 submitter: rishistyping (1)","date":"2026-06-01","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Aristotle","tool":"GPT-5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5 Codex + Aristotle\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Autoform-Bot","description":"1 submitter: niketp03 (1)","date":"2026-06-02","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Autoform-Bot","evidence":"lean-lang.org/eval leaderboard.json: \"Autoform-Bot\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Leanstral 1.4","description":"1 submitter: albertqjiang (1)","date":"2026-06-06","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Leanstral 1.4","evidence":"lean-lang.org/eval leaderboard.json: \"Leanstral 1.4\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions","description":"1 submitter: rkirov (1)","date":"2026-06-11","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OSS contributions","tool":"Claude Opus 4.7, 4.8 and Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Community multi-model project","description":"1 submitter: mrdouglasny (1)","date":"2026-06-15","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Community multi-model project","evidence":"lean-lang.org/eval leaderboard.json: \"Community multi-model project\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Codex (with human in the loop)","description":"1 submitter: plby (1)","date":"2026-06-26","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Codex (with human in the loop)","evidence":"lean-lang.org/eval leaderboard.json: \"Codex (with human in the loop)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.8 + Fable 5 (multi-agent)","description":"1 submitter: ldct (1)","date":"2026-07-05","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Fable 5 (multi-agent)","tool":"Claude Opus 4.8","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.8 + Fable 5 (multi-agent)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Hy3","description":"1 submitter: rwst (1)","date":"2026-07-12","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Hy3","evidence":"lean-lang.org/eval leaderboard.json: \"Hy3\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"UNICO/NOUS pipeline - Claude (Anthropic)","description":"1 submitter: Solarys431 (1)","date":"2026-07-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"UNICO/NOUS pipeline - Claude (Anthropic)","evidence":"lean-lang.org/eval leaderboard.json: \"UNICO/NOUS pipeline - Claude (Anthropic)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","description":"1 submitter: ianklatzco (1)","date":"2026-07-14","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 (human-in-the-loop)","description":"1 submitter: matthewjasper (1)","date":"2026-07-14","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 (human-in-the-loop)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 (human-in-the-loop)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Various models ( with human in the loop )","description":"1 submitter: yawara (1)","date":"2026-07-16","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Various models ( with human in the loop )","evidence":"lean-lang.org/eval leaderboard.json: \"Various models ( with human in the loop )\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Trellis","description":"1 submitter: wpegden (1)","date":"2026-07-20","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Trellis","evidence":"lean-lang.org/eval leaderboard.json: \"Trellis\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol","description":"1 submitter: Solarys431 (1)","date":"2026-07-21","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Opus 4.8) + GPT-5.6-Sol","tool":"UNICO/NOUS: Claude (Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"various model (deepseek-v4, gpt-5.*)","description":"1 submitter: ivwumupy (1)","date":"2026-07-24","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"various model (deepseek-v4, gpt-5.*)","evidence":"lean-lang.org/eval leaderboard.json: \"various model (deepseek-v4, gpt-5.*)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Fable 5 and GPT-5.6","description":"1 submitter: adamtopaz (1)","date":"2026-07-25","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Fable 5 and GPT-5.6","evidence":"lean-lang.org/eval leaderboard.json: \"Fable 5 and GPT-5.6\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","description":"1 submitter: rigwild (1)","date":"2026-08-04","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Codex (public-proof reproduction)","description":"1 submitter: Vilin97 (1)","date":"2026-08-07","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 Codex (public-proof reproduction)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 Codex (public-proof reproduction)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5 Codex","description":"1 submitter: Vilin97 (1)","date":"2026-08-12","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5 Codex\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-5)","description":"1 submitter: Vilin97 (1)","date":"2026-08-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OpenAI Codex (GPT-5)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Fable 5 (Claude Code)","description":"1 submitter: homeowmorphism (1)","date":"2026-08-18","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Fable 5 (Claude Code)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5 (Claude Code)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","description":"1 submitter: mlgraham (1)","date":"2026-08-20","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"DeepScientist + GPT-5.6","description":"1 submitter: LikC1606 (1)","date":"2026-08-22","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6","tool":"DeepScientist","evidence":"lean-lang.org/eval leaderboard.json: \"DeepScientist + GPT-5.6\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-5.6 Sol)","description":"1 submitter: Cond-t (1)","date":"2026-08-31","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OpenAI Codex (GPT-5.6 Sol)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5.6 Sol)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","description":"1 submitter: kim-em (1)","date":"2026-09-02","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","evidence":"lean-lang.org/eval leaderboard.json: \"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"BRILLiant + ChatGPT 5.6 Sol","description":"1 submitter: LexBrill (1)","date":"2026-09-02","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"ChatGPT 5.6 Sol","tool":"BRILLiant","evidence":"lean-lang.org/eval leaderboard.json: \"BRILLiant + ChatGPT 5.6 Sol\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-6 + GPT-5.6 Sol)","description":"1 submitter: lixiang90 (1)","date":"2026-09-07","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 Sol)","tool":"OpenAI Codex (GPT-6","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-6 + GPT-5.6 Sol)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"(various models)","description":"1 submitter: mlgraham (1)","date":"2026-09-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"(various models)","evidence":"lean-lang.org/eval leaderboard.json: \"(various models)\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"LeanEval P1 launch canary","description":"1 submitter: kim-em (1)","date":"2026-09-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"LeanEval P1 launch canary","evidence":"lean-lang.org/eval leaderboard.json: \"LeanEval P1 launch canary\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Codex 5.6 Sol + HITL","description":"1 submitter: kimihiro64 (1)","date":"2026-09-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"HITL","tool":"Codex 5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"Codex 5.6 Sol + HITL\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Fable 5.1 and GPT-6","description":"1 submitter: will1491 (1)","date":"2026-09-26","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Fable 5.1 and GPT-6","evidence":"lean-lang.org/eval leaderboard.json: \"Fable 5.1 and GPT-6\" solved 1","family":"anthropic","confidence":"high","role":"subject"}}],"contested":true},{"description":"People who ran a model and got a proof accepted, ranked by distinct problems solved. Attribution is the declared model on their accepted results.","id":"submitters","name":"Submitters — problems solved","metricName":"problems solved","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Vilin97","description":"models: Axiom Prover (Axiom Math) ×261, Vasily-24-hour-gpt-5.6-speedrun ×79, Opus-5 ×6, gpt-5.6-Sol ×5, GPT-5 Codex ×1, GPT-5.6 Codex (public-proof reproduction) ×1, OpenAI Codex (GPT-5) ×1","date":"2026-10-13","value":261,"contributorKind":"person","contributorUrl":"https://github.com/Vilin97","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Vilin97.json","https://github.com/Vilin97/lean-eval-speedrun","https://github.com/Vilin97/lean-eval-ado"],"isCurrentBest":true,"agent":{"model":"Axiom Prover (Axiom Math)","evidence":"results/Vilin97.json declared_model=\"Axiom Prover (Axiom Math)\" on 261 of 261 problems","family":"axiom","confidence":"high","role":"author"}},{"contributor":"ZhengyangZhang06","description":"models: Humanifa + GPT 5.6 sol ×215, Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5 ×7, Humanize RLCR: GPT-5.6-sol (max) worker + reviewer ×2","date":"2026-09-25","value":215,"contributorKind":"person","contributorUrl":"https://github.com/ZhengyangZhang06","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ZhengyangZhang06.json","https://github.com/humanfia/lean-eval"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/ZhengyangZhang06.json declared_model=\"Humanifa + GPT 5.6 sol\" on 215 of 215 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"lukerj00","description":"models: Tau (caj.al) ×200","date":"2026-09-07","value":200,"contributorKind":"person","contributorUrl":"https://github.com/lukerj00","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/lukerj00.json","https://github.com/lukerj00/tau-lean-eval-submission"],"agent":{"model":"Tau (caj.al)","evidence":"results/lukerj00.json declared_model=\"Tau (caj.al)\" on 200 of 200 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"GanjinZero","description":"models: Seed Prover (ByteDance) ×135","date":"2026-08-02","value":135,"contributorKind":"person","contributorUrl":"https://github.com/GanjinZero","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/GanjinZero.json","https://github.com/GanjinZero/lidskii_last","https://github.com/GanjinZero/frobenius_group_determinant"],"agent":{"model":"Seed Prover (ByteDance)","evidence":"results/GanjinZero.json declared_model=\"Seed Prover (ByteDance)\" on 135 of 135 problems","family":"bytedance","confidence":"high","role":"author"}},{"contributor":"Morgan-Griffiths","description":"models: GPT-5.6 ×111, GPT-5.5 ×10","date":"2026-07-16","value":111,"contributorKind":"person","contributorUrl":"https://github.com/Morgan-Griffiths","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Morgan-Griffiths.json","https://github.com/Morgan-Griffiths/4bdee1299a2cb3e5ad3ee11129659885","https://github.com/Morgan-Griffiths/540d938ce971aabb2511aae2ea81e911"],"agent":{"model":"GPT-5.6","evidence":"results/Morgan-Griffiths.json declared_model=\"GPT-5.6\" on 111 of 111 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"LorenzoLuccioli","description":"models: Aristotle (Harmonic) ×105","date":"2026-07-18","value":105,"contributorKind":"person","contributorUrl":"https://github.com/LorenzoLuccioli","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LorenzoLuccioli.json","https://github.com/LorenzoLuccioli/lean-eval-private","https://github.com/LorenzoLuccioli/lean-eval"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/LorenzoLuccioli.json declared_model=\"Aristotle (Harmonic)\" on 105 of 105 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"rishistyping","description":"models: Stealth Model ×104, Public accepted source + Codex packaging ×7, [submission] aegis-of-the-unit-circle-logos ×2, GPT-5 Codex + Aristotle ×1","date":"2026-08-20","value":104,"contributorKind":"person","contributorUrl":"https://github.com/rishistyping","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rishistyping.json","https://github.com/rishistyping/autoformalization-machine-intelligence","https://github.com/rishistyping/lean-eval-symplectic-matrix-det-submission"],"agent":{"model":"Stealth Model","evidence":"results/rishistyping.json declared_model=\"Stealth Model\" on 104 of 104 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"skidanovalex","description":"models: NEAR AI w/ DeepSeek V4 ×85","date":"2026-10-13","value":85,"contributorKind":"person","contributorUrl":"https://github.com/skidanovalex","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/skidanovalex.json","https://github.com/SkidanovAlex/leaneval-deepseek"],"agent":{"model":"DeepSeek V4","tool":"NEAR AI","evidence":"results/skidanovalex.json declared_model=\"NEAR AI w/ DeepSeek V4\" on 85 of 85 problems","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"ma4ypic4y","description":"models: max gp (GPT-6) ×73","date":"2026-10-13","value":73,"contributorKind":"person","contributorUrl":"https://github.com/ma4ypic4y","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ma4ypic4y.json","https://github.com/ma4ypic4y/lean-eval-top1"],"agent":{"model":"max gp (GPT-6)","evidence":"results/ma4ypic4y.json declared_model=\"max gp (GPT-6)\" on 73 of 73 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mayorov-m-a","description":"models: Aleph Prover(logicalintelligence.com) ×52","date":"2026-06-05","value":52,"contributorKind":"person","contributorUrl":"https://github.com/mayorov-m-a","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mayorov-m-a.json","https://github.com/mayorov-m-a/aleph-lean-eval-subm-2026-may-08"],"agent":{"model":"Aleph Prover(logicalintelligence.com)","evidence":"results/mayorov-m-a.json declared_model=\"Aleph Prover(logicalintelligence.com)\" on 52 of 52 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"parabamoghv","description":"models: Aristotle (Harmonic) ×42","date":"2026-08-20","value":42,"contributorKind":"person","contributorUrl":"https://github.com/parabamoghv","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/parabamoghv.json","https://github.com/parabamoghv/lean-eval","https://github.com/parabamoghv/lean-eval-private"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/parabamoghv.json declared_model=\"Aristotle (Harmonic)\" on 42 of 42 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"sarjinius","description":"models: Midas ×39","date":"2026-10-01","value":39,"contributorKind":"person","contributorUrl":"https://github.com/sarjinius","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/sarjinius.json","https://github.com/midas-intelligence/midas-leaneval-submission"],"agent":{"model":"Midas","evidence":"results/sarjinius.json declared_model=\"Midas\" on 39 of 39 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"hongzhoulin89","description":"models: Humanifa + GPT 5.6 sol ×34, Humanifa + GPT-5.6 sol ×2","date":"2026-10-04","value":34,"contributorKind":"person","contributorUrl":"https://github.com/hongzhoulin89","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/hongzhoulin89.json","https://github.com/hongzhoulin89/lean-eval-bourgain-polynomial-ergodic","https://github.com/hongzhoulin89/lean-eval-pardon-torus-knot-distortion"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/hongzhoulin89.json declared_model=\"Humanifa + GPT 5.6 sol\" on 34 of 34 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mlgraham","description":"models: github.com/mlgraham (various models) ×31, github.com/mlgraham (Claude Fable 5) ×16, github.com/mlgraham — public accepted source, packaged (Claude Fable 5) ×1, (various models) ×1","date":"2026-09-13","value":32,"contributorKind":"person","contributorUrl":"https://github.com/mlgraham","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mlgraham.json","https://github.com/mlgraham/lean-eval-submission","https://github.com/mlgraham/lean-eval-coc"],"agent":{"model":"github.com/mlgraham (various models)","evidence":"results/mlgraham.json declared_model=\"github.com/mlgraham (various models)\" on 31 of 32 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"doxtor6","description":"models: MerLean-Prover ×30","date":"2026-06-25","value":30,"contributorKind":"person","contributorUrl":"https://github.com/doxtor6","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/doxtor6.json","https://github.com/doxtor6/lean-eval"],"agent":{"model":"MerLean-Prover","evidence":"results/doxtor6.json declared_model=\"MerLean-Prover\" on 30 of 30 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"daouid","description":"models: Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus) ×21","date":"2026-05-22","value":21,"contributorKind":"person","contributorUrl":"https://github.com/daouid","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/daouid.json","https://github.com/daouid/lean-eval","https://github.com/daouid/73182095262a4e329bcec8579bdd573a"],"agent":{"model":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","evidence":"results/daouid.json declared_model=\"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)\" on 21 of 21 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"hanwenzhu","description":"models: Seed Prover (ByteDance) ×21","date":"2026-08-13","value":21,"contributorKind":"person","contributorUrl":"https://github.com/hanwenzhu","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/hanwenzhu.json","https://github.com/hanwenzhu/lean-eval-seedprover"],"agent":{"model":"Seed Prover (ByteDance)","evidence":"results/hanwenzhu.json declared_model=\"Seed Prover (ByteDance)\" on 21 of 21 problems","family":"bytedance","confidence":"high","role":"author"}},{"contributor":"savarin","description":"models: savarin-6-hour-gpt-5.6-speedrun ×21","date":"2026-08-17","value":21,"contributorKind":"person","contributorUrl":"https://github.com/savarin","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/savarin.json","https://github.com/savarin/lean-eval-speedrun"],"agent":{"model":"savarin-6-hour-gpt-5.6-speedrun","evidence":"results/savarin.json declared_model=\"savarin-6-hour-gpt-5.6-speedrun\" on 21 of 21 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"rkirov","description":"models: Claude Opus 4.7 (1M context) ×15, Claude Fable 5 ×4, Claude Opus 4.8 (1M context) ×2, Claude Opus 4.7 ×1, Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions ×1","date":"2026-07-10","value":19,"contributorKind":"person","contributorUrl":"https://github.com/rkirov","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rkirov.json","https://github.com/rkirov/lean-eval","https://github.com/rkirov/jordan_pick"],"agent":{"model":"Claude Opus 4.7 (1M context)","evidence":"results/rkirov.json declared_model=\"Claude Opus 4.7 (1M context)\" on 15 of 19 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"sqrt-of-2","description":"models: Aristotle (Harmonic) ×7, GPT-5.5 ×7, GPT 5.5 ×2, Gemini 3.1 Pro ×2","date":"2026-05-12","value":11,"contributorKind":"person","contributorUrl":"https://github.com/sqrt-of-2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/sqrt-of-2.json","https://github.com/sqrt-of-2/931556b393de1dc23de63d1c5aefc109","https://github.com/sqrt-of-2/5d253653afb78ab2e57ae1446599c9eb"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/sqrt-of-2.json declared_model=\"Aristotle (Harmonic)\" on 7 of 11 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"test1-deepthought","description":"models: EVO (deepthought.com.au) ×8, EVO ×2","date":"2026-06-23","value":8,"contributorKind":"person","contributorUrl":"https://github.com/test1-deepthought","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/test1-deepthought.json","https://github.com/test1-deepthought/lean-eval-solutions","https://github.com/machinelearning2014/deepthought_lean_eval"],"agent":{"model":"EVO (deepthought.com.au)","evidence":"results/test1-deepthought.json declared_model=\"EVO (deepthought.com.au)\" on 8 of 8 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"AlyciaBHZ","description":"models: trureturing ×7, trureturning ×2","date":"2026-10-05","value":7,"contributorKind":"person","contributorUrl":"https://github.com/AlyciaBHZ","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/AlyciaBHZ.json","https://github.com/the-omega-institute/trureturning-lean-eval","https://github.com/the-omega-institute/trureturing-lean-eval"],"agent":{"model":"trureturing","evidence":"results/AlyciaBHZ.json declared_model=\"trureturing\" on 7 of 7 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"JohnEdwardJennings","description":"models: Aristotle (Harmonic) ×7","date":"2026-06-14","value":7,"contributorKind":"person","contributorUrl":"https://github.com/JohnEdwardJennings","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/JohnEdwardJennings.json","https://github.com/JohnEdwardJennings/lean-eval-personal"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/JohnEdwardJennings.json declared_model=\"Aristotle (Harmonic)\" on 7 of 7 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"kim-em","description":"models: Aristotle (Harmonic) ×6, LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183 ×1, LeanEval P1 launch canary ×1","date":"2026-09-13","value":7,"contributorKind":"person","contributorUrl":"https://github.com/kim-em","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/kim-em.json","https://github.com/kim-em/0e2146482105e90a2bb4b63f137975fb","https://github.com/kim-em/ba837b9520ea4ec64c7945bb1f0cf10f"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/kim-em.json declared_model=\"Aristotle (Harmonic)\" on 6 of 7 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"juihuichung","description":"models: Humanifa + GPT 5.6 sol ×6","date":"2026-08-31","value":6,"contributorKind":"person","contributorUrl":"https://github.com/juihuichung","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/juihuichung.json","https://github.com/juihuichung/lean-eval-submissions-private"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/juihuichung.json declared_model=\"Humanifa + GPT 5.6 sol\" on 6 of 6 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"antoinegg1","description":"models: Humanifa + GPT 5.6 sol ×5","date":"2026-10-13","value":5,"contributorKind":"person","contributorUrl":"https://github.com/antoinegg1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/antoinegg1.json","https://github.com/antoinegg1/lean-eval-lot-solutions"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/antoinegg1.json declared_model=\"Humanifa + GPT 5.6 sol\" on 5 of 5 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"xiaoyangliu-sjtu","description":"models: Forge ×5","date":"2026-09-15","value":5,"contributorKind":"person","contributorUrl":"https://github.com/xiaoyangliu-sjtu","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/xiaoyangliu-sjtu.json","https://github.com/XiaoyangLiu-sjtu/forge-leaneval-submissions"],"agent":{"model":"Forge","evidence":"results/xiaoyangliu-sjtu.json declared_model=\"Forge\" on 5 of 5 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"A-M-Berns","description":"models: GPT-5.5 ×4, GPT-5.5 Codex ×2","date":"2026-05-16","value":4,"contributorKind":"person","contributorUrl":"https://github.com/A-M-Berns","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/A-M-Berns.json","https://github.com/A-M-Berns/lean-eval-submissions"],"agent":{"model":"GPT-5.5","evidence":"results/A-M-Berns.json declared_model=\"GPT-5.5\" on 4 of 4 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"antpavzhi","description":"models: Aleph Prover(logicalintelligence.com) ×4","date":"2026-05-11","value":4,"contributorKind":"person","contributorUrl":"https://github.com/antpavzhi","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/antpavzhi.json","https://github.com/antpavzhi/lean-eval"],"agent":{"model":"Aleph Prover(logicalintelligence.com)","evidence":"results/antpavzhi.json declared_model=\"Aleph Prover(logicalintelligence.com)\" on 4 of 4 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"bogatyy","description":"models: Ivan Bogatyy ×4","date":"2026-09-19","value":4,"contributorKind":"person","contributorUrl":"https://github.com/bogatyy","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/bogatyy.json","https://github.com/bogatyy/lean-eval-proofs-intake"],"agent":{"model":"Ivan Bogatyy","evidence":"results/bogatyy.json declared_model=\"Ivan Bogatyy\" on 4 of 4 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"adrianmartir","description":"models: Aristotle (Harmonic) ×3","date":"2026-06-29","value":3,"contributorKind":"person","contributorUrl":"https://github.com/adrianmartir","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/adrianmartir.json","https://github.com/adrianmartir/lean-eval-private","https://github.com/LorenzoLuccioli/lean-eval-private"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/adrianmartir.json declared_model=\"Aristotle (Harmonic)\" on 3 of 3 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"eohjelle","description":"models: GPT-5.6 ×3","date":"2026-07-21","value":3,"contributorKind":"person","contributorUrl":"https://github.com/eohjelle","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/eohjelle.json","https://github.com/eohjelle/feit-thompson-odd-order-theorem"],"agent":{"model":"GPT-5.6","evidence":"results/eohjelle.json declared_model=\"GPT-5.6\" on 3 of 3 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"ivwumupy","description":"models: deepseek-v4 and gpt-5.x ×3, various model (deepseek-v4, gpt-5.*) ×1","date":"2026-07-27","value":3,"contributorKind":"person","contributorUrl":"https://github.com/ivwumupy","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ivwumupy.json","https://github.com/Qiuzhen-CFSG/CFSG"],"agent":{"model":"deepseek-v4 and gpt-5.x","evidence":"results/ivwumupy.json declared_model=\"deepseek-v4 and gpt-5.x\" on 3 of 3 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mccorvie","description":"models: mostly ChatGPT 5.6 Sol ×3","date":"2026-08-03","value":3,"contributorKind":"person","contributorUrl":"https://github.com/mccorvie","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mccorvie.json","https://github.com/mccorvie/classification-of-surfaces","https://github.com/mccorvie/tcos-submission"],"agent":{"model":"mostly ChatGPT 5.6 Sol","evidence":"results/mccorvie.json declared_model=\"mostly ChatGPT 5.6 Sol\" on 3 of 3 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"rwst","description":"models: Grok 4.5 ×3, DeepSeek V4 Flash ×3, Kimi K2.7 ×2, Hy3 ×1","date":"2026-07-23","value":3,"contributorKind":"person","contributorUrl":"https://github.com/rwst","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rwst.json","https://github.com/rwst/lean-eval"],"agent":{"model":"Grok 4.5","evidence":"results/rwst.json declared_model=\"Grok 4.5\" on 3 of 3 problems","family":"xai","confidence":"high","role":"author"}},{"contributor":"jzuiddam","description":"models: Claude Opus 4.7 (1M context) ×2","date":"2026-05-07","value":2,"contributorKind":"person","contributorUrl":"https://github.com/jzuiddam","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/jzuiddam.json","https://github.com/jzuiddam/symAction-eval-submission","https://github.com/jzuiddam/glAction-eval-submission"],"agent":{"model":"Claude Opus 4.7 (1M context)","evidence":"results/jzuiddam.json declared_model=\"Claude Opus 4.7 (1M context)\" on 2 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"KitaKen1","description":"models: GPT-5.6 and Fable 5 (human-in-the-loop) ×2","date":"2026-08-06","value":2,"contributorKind":"person","contributorUrl":"https://github.com/KitaKen1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/KitaKen1.json","https://github.com/KitaKen1/lean-eval-pi-succ-sphere-n-mulequiv-zmod-two","https://github.com/KitaKen1/lean-eval-conway-schneeberger-fifteen"],"agent":{"model":"GPT-5.6 and Fable 5 (human-in-the-loop)","evidence":"results/KitaKen1.json declared_model=\"GPT-5.6 and Fable 5 (human-in-the-loop)\" on 2 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"machinelearning2014","description":"models: EVO ×2","date":"2026-05-30","value":2,"contributorKind":"person","contributorUrl":"https://github.com/machinelearning2014","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/machinelearning2014.json","https://github.com/machinelearning2014/deepthought_lean_eval"],"agent":{"model":"EVO","evidence":"results/machinelearning2014.json declared_model=\"EVO\" on 2 of 2 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"n-yamaguchi-0729","description":"models: GPT-5.6 Sol ×2","date":"2026-09-08","value":2,"contributorKind":"person","contributorUrl":"https://github.com/n-yamaguchi-0729","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/n-yamaguchi-0729.json","https://github.com/n-yamaguchi-0729/LeanEvalSubmissions"],"agent":{"model":"GPT-5.6 Sol","evidence":"results/n-yamaguchi-0729.json declared_model=\"GPT-5.6 Sol\" on 2 of 2 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"nolanlwin","description":"models: Forall (Astrio) ×2","date":"2026-09-07","value":2,"contributorKind":"person","contributorUrl":"https://github.com/nolanlwin","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/nolanlwin.json","https://github.com/nolanlwin/lean-eval"],"agent":{"model":"Forall (Astrio)","evidence":"results/nolanlwin.json declared_model=\"Forall (Astrio)\" on 2 of 2 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"Solarys431","description":"models: UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol ×1, UNICO/NOUS pipeline - Claude (Anthropic) ×1","date":"2026-07-21","value":2,"contributorKind":"person","contributorUrl":"https://github.com/Solarys431","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Solarys431.json","https://github.com/Solarys431/lean-eval-platonic-classification","https://github.com/Solarys431/unico-lean-proofs"],"agent":{"model":"Opus 4.8) + GPT-5.6-Sol","tool":"UNICO/NOUS: Claude (Fable 5","evidence":"results/Solarys431.json declared_model=\"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol\" on 1 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"will1491","description":"models: Claude Fable 5 ×1, Fable 5.1 and GPT-6 ×1","date":"2026-09-26","value":2,"contributorKind":"person","contributorUrl":"https://github.com/will1491","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/will1491.json","https://github.com/will1491/uniformization","https://github.com/will1491/Poincare-High-Dim"],"agent":{"model":"Claude Fable 5","evidence":"results/will1491.json declared_model=\"Claude Fable 5\" on 1 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"yawara","description":"models: Various models ( with human in the loop ) ×1, Claude Opus 4.7 + GPT-5.5 (human-in-the-loop) ×1","date":"2026-07-16","value":2,"contributorKind":"person","contributorUrl":"https://github.com/yawara","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/yawara.json","https://github.com/yawara/odd-order-submission","https://github.com/yawara/lean-eval-submissions"],"agent":{"model":"Various models ( with human in the loop )","evidence":"results/yawara.json declared_model=\"Various models ( with human in the loop )\" on 1 of 2 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"adamtopaz","description":"models: Fable 5 and GPT-5.6 ×1","date":"2026-07-25","value":1,"contributorKind":"person","contributorUrl":"https://github.com/adamtopaz","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/adamtopaz.json","https://github.com/adamtopaz/NeukirchUchida"],"agent":{"model":"Fable 5 and GPT-5.6","evidence":"results/adamtopaz.json declared_model=\"Fable 5 and GPT-5.6\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"albertqjiang","description":"models: Leanstral 1.4 ×1","date":"2026-06-06","value":1,"contributorKind":"person","contributorUrl":"https://github.com/albertqjiang","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/albertqjiang.json","https://github.com/albertqjiang/lean-eval-submissions"],"agent":{"model":"Leanstral 1.4","evidence":"results/albertqjiang.json declared_model=\"Leanstral 1.4\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"Cond-t","description":"models: OpenAI Codex (GPT-5.6 Sol) ×1","date":"2026-08-31","value":1,"contributorKind":"person","contributorUrl":"https://github.com/Cond-t","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Cond-t.json","https://github.com/Cond-t/thema-chen-theorem-lean-eval"],"agent":{"model":"OpenAI Codex (GPT-5.6 Sol)","evidence":"results/Cond-t.json declared_model=\"OpenAI Codex (GPT-5.6 Sol)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"homeowmorphism","description":"models: Claude Fable 5 (Claude Code) ×1","date":"2026-08-18","value":1,"contributorKind":"person","contributorUrl":"https://github.com/homeowmorphism","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/homeowmorphism.json","https://github.com/homeowmorphism/lean-eval-annals-marton"],"agent":{"model":"Claude Fable 5 (Claude Code)","evidence":"results/homeowmorphism.json declared_model=\"Claude Fable 5 (Claude Code)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"ianklatzco","description":"models: Claude Fable 5 (orchestrating Opus/Sonnet 4.8) ×1","date":"2026-07-14","value":1,"contributorKind":"person","contributorUrl":"https://github.com/ianklatzco","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ianklatzco.json","https://github.com/ianklatzco/odd-order-lean"],"agent":{"model":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","evidence":"results/ianklatzco.json declared_model=\"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"kimihiro64","description":"models: Codex 5.6 Sol + HITL ×1","date":"2026-09-13","value":1,"contributorKind":"person","contributorUrl":"https://github.com/kimihiro64","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/kimihiro64.json","https://github.com/kimihiro64/Robin1984"],"agent":{"model":"HITL","tool":"Codex 5.6 Sol","evidence":"results/kimihiro64.json declared_model=\"Codex 5.6 Sol + HITL\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"ldct","description":"models: Claude Opus 4.8 + Fable 5 (multi-agent) ×1","date":"2026-07-05","value":1,"contributorKind":"person","contributorUrl":"https://github.com/ldct","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ldct.json","https://github.com/ldct/lean-eval-chudnovsky"],"agent":{"model":"Fable 5 (multi-agent)","tool":"Claude Opus 4.8","evidence":"results/ldct.json declared_model=\"Claude Opus 4.8 + Fable 5 (multi-agent)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"LexBrill","description":"models: BRILLiant + ChatGPT 5.6 Sol ×1","date":"2026-09-02","value":1,"contributorKind":"person","contributorUrl":"https://github.com/LexBrill","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LexBrill.json","https://github.com/LexBrill/lean-eval-annals-unit-submission"],"agent":{"model":"ChatGPT 5.6 Sol","tool":"BRILLiant","evidence":"results/LexBrill.json declared_model=\"BRILLiant + ChatGPT 5.6 Sol\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"LikC1606","description":"models: DeepScientist + GPT-5.6 ×1","date":"2026-08-22","value":1,"contributorKind":"person","contributorUrl":"https://github.com/LikC1606","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LikC1606.json","https://github.com/LikC1606/7570e284ff8b474a4139db2092de6eec"],"agent":{"model":"GPT-5.6","tool":"DeepScientist","evidence":"results/LikC1606.json declared_model=\"DeepScientist + GPT-5.6\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"lixiang90","description":"models: OpenAI Codex (GPT-6 + GPT-5.6 Sol) ×1","date":"2026-09-07","value":1,"contributorKind":"person","contributorUrl":"https://github.com/lixiang90","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/lixiang90.json","https://github.com/lixiang90/chen_theorem"],"agent":{"model":"GPT-5.6 Sol)","tool":"OpenAI Codex (GPT-6","evidence":"results/lixiang90.json declared_model=\"OpenAI Codex (GPT-6 + GPT-5.6 Sol)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"matthewjasper","description":"models: GPT-5.6 (human-in-the-loop) ×1","date":"2026-07-14","value":1,"contributorKind":"person","contributorUrl":"https://github.com/matthewjasper","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/matthewjasper.json","https://github.com/matthewjasper/m23_irrep_tensor_square_decomp"],"agent":{"model":"GPT-5.6 (human-in-the-loop)","evidence":"results/matthewjasper.json declared_model=\"GPT-5.6 (human-in-the-loop)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mrdouglasny","description":"models: Community multi-model project ×1","date":"2026-06-15","value":1,"contributorKind":"person","contributorUrl":"https://github.com/mrdouglasny","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mrdouglasny.json","https://github.com/mrdouglasny/jacobian-challenge"],"agent":{"model":"Community multi-model project","evidence":"results/mrdouglasny.json declared_model=\"Community multi-model project\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"niketp03","description":"models: Autoform-Bot ×1","date":"2026-06-02","value":1,"contributorKind":"person","contributorUrl":"https://github.com/niketp03","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/niketp03.json","https://github.com/niketp03/fourier-dirichlet-fejer-submission"],"agent":{"model":"Autoform-Bot","evidence":"results/niketp03.json declared_model=\"Autoform-Bot\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"Parcly-Taxel","description":"models: Aristotle (Harmonic) ×1","date":"2026-05-23","value":1,"contributorKind":"person","contributorUrl":"https://github.com/Parcly-Taxel","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Parcly-Taxel.json","https://github.com/Parcly-Taxel/lean-eval-bbp"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/Parcly-Taxel.json declared_model=\"Aristotle (Harmonic)\" on 1 of 1 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"plby","description":"models: Codex (with human in the loop) ×1","date":"2026-06-26","value":1,"contributorKind":"person","contributorUrl":"https://github.com/plby","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/plby.json","https://github.com/plby/Erdos90"],"agent":{"model":"Codex (with human in the loop)","evidence":"results/plby.json declared_model=\"Codex (with human in the loop)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"rigwild","description":"models: OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8) ×1","date":"2026-08-04","value":1,"contributorKind":"person","contributorUrl":"https://github.com/rigwild","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rigwild.json","https://github.com/rigwild/lean-math-problems"],"agent":{"model":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","evidence":"results/rigwild.json declared_model=\"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"wpegden","description":"models: Trellis ×1","date":"2026-07-20","value":1,"contributorKind":"person","contributorUrl":"https://github.com/wpegden","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/wpegden.json","https://github.com/wpegden/unitdistance_challenge"],"agent":{"model":"Trellis","evidence":"results/wpegden.json declared_model=\"Trellis\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}}],"contested":true}],"agentStats":[{"family":"axiom","records":1,"currentBests":1,"models":["Axiom Prover (Axiom Math)"],"contributors":["Vilin97"]},{"family":"openai","records":19,"currentBests":0,"models":["GPT 5.6 sol","GPT-5.6","max gp (GPT-6)","savarin-6-hour-gpt-5.6-speedrun","GPT-5.5","deepseek-v4 and gpt-5.x","mostly ChatGPT 5.6 Sol","GPT-5.6 Sol","OpenAI Codex (GPT-5.6 Sol)","HITL","ChatGPT 5.6 Sol","GPT-5.6 Sol)","GPT-5.6 (human-in-the-loop)","Codex (with human in the loop)"],"contributors":["ZhengyangZhang06","Morgan-Griffiths","ma4ypic4y","hongzhoulin89","savarin","juihuichung","antoinegg1","A-M-Berns","eohjelle","ivwumupy","mccorvie","n-yamaguchi-0729","Cond-t","kimihiro64","LexBrill","LikC1606","lixiang90","matthewjasper","plby"]},{"family":"other-ai","records":18,"currentBests":0,"models":["Tau (caj.al)","Stealth Model","Aleph Prover(logicalintelligence.com)","Midas","github.com/mlgraham (various models)","MerLean-Prover","EVO (deepthought.com.au)","trureturing","Forge","Ivan Bogatyy","EVO","Forall (Astrio)","Various models ( with human in the loop )","Leanstral 1.4","Community multi-model project","Autoform-Bot","Trellis"],"contributors":["lukerj00","rishistyping","mayorov-m-a","sarjinius","mlgraham","doxtor6","test1-deepthought","AlyciaBHZ","xiaoyangliu-sjtu","antpavzhi","bogatyy","machinelearning2014","nolanlwin","yawara","albertqjiang","mrdouglasny","niketp03","wpegden"]},{"family":"anthropic","records":11,"currentBests":0,"models":["Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","Claude Opus 4.7 (1M context)","GPT-5.6 and Fable 5 (human-in-the-loop)","Opus 4.8) + GPT-5.6-Sol","Claude Fable 5","Fable 5 and GPT-5.6","Claude Fable 5 (Claude Code)","Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","Fable 5 (multi-agent)","OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)"],"contributors":["daouid","rkirov","jzuiddam","KitaKen1","Solarys431","will1491","adamtopaz","homeowmorphism","ianklatzco","ldct","rigwild"]},{"family":"harmonic","records":7,"currentBests":0,"models":["Aristotle (Harmonic)"],"contributors":["LorenzoLuccioli","parabamoghv","sqrt-of-2","JohnEdwardJennings","kim-em","adrianmartir","Parcly-Taxel"]},{"family":"bytedance","records":2,"currentBests":0,"models":["Seed Prover (ByteDance)"],"contributors":["GanjinZero","hanwenzhu"]},{"family":"deepseek","records":1,"currentBests":0,"models":["DeepSeek V4"],"contributors":["skidanovalex"]},{"family":"xai","records":1,"currentBests":0,"models":["Grok 4.5"],"contributors":["rwst"]}],"repo":{"owner":"leanprover","name":"lean-eval-submissions","branch":"main"},"participants":[{"name":"Vilin97","url":"https://github.com/Vilin97","submissions":1,"problems":["submitters"],"bestRank":1,"lastActive":"2026-10-13"},{"name":"ZhengyangZhang06","url":"https://github.com/ZhengyangZhang06","submissions":1,"problems":["submitters"],"lastActive":"2026-09-25"},{"name":"lukerj00","url":"https://github.com/lukerj00","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"GanjinZero","url":"https://github.com/GanjinZero","submissions":1,"problems":["submitters"],"lastActive":"2026-08-02"},{"name":"Morgan-Griffiths","url":"https://github.com/Morgan-Griffiths","submissions":1,"problems":["submitters"],"lastActive":"2026-07-16"},{"name":"LorenzoLuccioli","url":"https://github.com/LorenzoLuccioli","submissions":1,"problems":["submitters"],"lastActive":"2026-07-18"},{"name":"rishistyping","url":"https://github.com/rishistyping","submissions":1,"problems":["submitters"],"lastActive":"2026-08-20"},{"name":"skidanovalex","url":"https://github.com/skidanovalex","submissions":1,"problems":["submitters"],"lastActive":"2026-10-13"},{"name":"ma4ypic4y","url":"https://github.com/ma4ypic4y","submissions":1,"problems":["submitters"],"lastActive":"2026-10-13"},{"name":"mayorov-m-a","url":"https://github.com/mayorov-m-a","submissions":1,"problems":["submitters"],"lastActive":"2026-06-05"},{"name":"parabamoghv","url":"https://github.com/parabamoghv","submissions":1,"problems":["submitters"],"lastActive":"2026-08-20"},{"name":"sarjinius","url":"https://github.com/sarjinius","submissions":1,"problems":["submitters"],"lastActive":"2026-10-01"},{"name":"hongzhoulin89","url":"https://github.com/hongzhoulin89","submissions":1,"problems":["submitters"],"lastActive":"2026-10-04"},{"name":"mlgraham","url":"https://github.com/mlgraham","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"doxtor6","url":"https://github.com/doxtor6","submissions":1,"problems":["submitters"],"lastActive":"2026-06-25"},{"name":"daouid","url":"https://github.com/daouid","submissions":1,"problems":["submitters"],"lastActive":"2026-05-22"},{"name":"hanwenzhu","url":"https://github.com/hanwenzhu","submissions":1,"problems":["submitters"],"lastActive":"2026-08-13"},{"name":"savarin","url":"https://github.com/savarin","submissions":1,"problems":["submitters"],"lastActive":"2026-08-17"},{"name":"rkirov","url":"https://github.com/rkirov","submissions":1,"problems":["submitters"],"lastActive":"2026-07-10"},{"name":"sqrt-of-2","url":"https://github.com/sqrt-of-2","submissions":1,"problems":["submitters"],"lastActive":"2026-05-12"},{"name":"test1-deepthought","url":"https://github.com/test1-deepthought","submissions":1,"problems":["submitters"],"lastActive":"2026-06-23"},{"name":"AlyciaBHZ","url":"https://github.com/AlyciaBHZ","submissions":1,"problems":["submitters"],"lastActive":"2026-10-05"},{"name":"JohnEdwardJennings","url":"https://github.com/JohnEdwardJennings","submissions":1,"problems":["submitters"],"lastActive":"2026-06-14"},{"name":"kim-em","url":"https://github.com/kim-em","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"juihuichung","url":"https://github.com/juihuichung","submissions":1,"problems":["submitters"],"lastActive":"2026-08-31"},{"name":"antoinegg1","url":"https://github.com/antoinegg1","submissions":1,"problems":["submitters"],"lastActive":"2026-10-13"},{"name":"xiaoyangliu-sjtu","url":"https://github.com/xiaoyangliu-sjtu","submissions":1,"problems":["submitters"],"lastActive":"2026-09-15"},{"name":"A-M-Berns","url":"https://github.com/A-M-Berns","submissions":1,"problems":["submitters"],"lastActive":"2026-05-16"},{"name":"antpavzhi","url":"https://github.com/antpavzhi","submissions":1,"problems":["submitters"],"lastActive":"2026-05-11"},{"name":"bogatyy","url":"https://github.com/bogatyy","submissions":1,"problems":["submitters"],"lastActive":"2026-09-19"},{"name":"adrianmartir","url":"https://github.com/adrianmartir","submissions":1,"problems":["submitters"],"lastActive":"2026-06-29"},{"name":"eohjelle","url":"https://github.com/eohjelle","submissions":1,"problems":["submitters"],"lastActive":"2026-07-21"},{"name":"ivwumupy","url":"https://github.com/ivwumupy","submissions":1,"problems":["submitters"],"lastActive":"2026-07-27"},{"name":"mccorvie","url":"https://github.com/mccorvie","submissions":1,"problems":["submitters"],"lastActive":"2026-08-03"},{"name":"rwst","url":"https://github.com/rwst","submissions":1,"problems":["submitters"],"lastActive":"2026-07-23"},{"name":"jzuiddam","url":"https://github.com/jzuiddam","submissions":1,"problems":["submitters"],"lastActive":"2026-05-07"},{"name":"KitaKen1","url":"https://github.com/KitaKen1","submissions":1,"problems":["submitters"],"lastActive":"2026-08-06"},{"name":"machinelearning2014","url":"https://github.com/machinelearning2014","submissions":1,"problems":["submitters"],"lastActive":"2026-05-30"},{"name":"n-yamaguchi-0729","url":"https://github.com/n-yamaguchi-0729","submissions":1,"problems":["submitters"],"lastActive":"2026-09-08"},{"name":"nolanlwin","url":"https://github.com/nolanlwin","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"Solarys431","url":"https://github.com/Solarys431","submissions":1,"problems":["submitters"],"lastActive":"2026-07-21"},{"name":"will1491","url":"https://github.com/will1491","submissions":1,"problems":["submitters"],"lastActive":"2026-09-26"},{"name":"yawara","url":"https://github.com/yawara","submissions":1,"problems":["submitters"],"lastActive":"2026-07-16"},{"name":"adamtopaz","url":"https://github.com/adamtopaz","submissions":1,"problems":["submitters"],"lastActive":"2026-07-25"},{"name":"albertqjiang","url":"https://github.com/albertqjiang","submissions":1,"problems":["submitters"],"lastActive":"2026-06-06"},{"name":"Cond-t","url":"https://github.com/Cond-t","submissions":1,"problems":["submitters"],"lastActive":"2026-08-31"},{"name":"homeowmorphism","url":"https://github.com/homeowmorphism","submissions":1,"problems":["submitters"],"lastActive":"2026-08-18"},{"name":"ianklatzco","url":"https://github.com/ianklatzco","submissions":1,"problems":["submitters"],"lastActive":"2026-07-14"},{"name":"kimihiro64","url":"https://github.com/kimihiro64","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"ldct","url":"https://github.com/ldct","submissions":1,"problems":["submitters"],"lastActive":"2026-07-05"},{"name":"LexBrill","url":"https://github.com/LexBrill","submissions":1,"problems":["submitters"],"lastActive":"2026-09-02"},{"name":"LikC1606","url":"https://github.com/LikC1606","submissions":1,"problems":["submitters"],"lastActive":"2026-08-22"},{"name":"lixiang90","url":"https://github.com/lixiang90","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"matthewjasper","url":"https://github.com/matthewjasper","submissions":1,"problems":["submitters"],"lastActive":"2026-07-14"},{"name":"mrdouglasny","url":"https://github.com/mrdouglasny","submissions":1,"problems":["submitters"],"lastActive":"2026-06-15"},{"name":"niketp03","url":"https://github.com/niketp03","submissions":1,"problems":["submitters"],"lastActive":"2026-06-02"},{"name":"Parcly-Taxel","url":"https://github.com/Parcly-Taxel","submissions":1,"problems":["submitters"],"lastActive":"2026-05-23"},{"name":"plby","url":"https://github.com/plby","submissions":1,"problems":["submitters"],"lastActive":"2026-06-26"},{"name":"rigwild","url":"https://github.com/rigwild","submissions":1,"problems":["submitters"],"lastActive":"2026-08-04"},{"name":"wpegden","url":"https://github.com/wpegden","submissions":1,"problems":["submitters"],"lastActive":"2026-07-20"}],"stats":{"totalRecords":135,"uniqueParticipants":60,"firstSubmission":"2026-04-30","lastSubmission":"2026-10-13","recordsLast30d":21,"recordsLast90d":88},"lastUpdated":"2026-10-13T12:55:04.142Z"},{"id":"s2n-bignum-bench","name":"s2n-bignum-bench","tagline":"Benchmark LLM low-level code reasoning via HOL Light tactic synthesis on AWS cryptographic proofs.","description":"A practical benchmark for evaluating low-level code reasoning of LLMs through HOL Light tactic synthesis on cryptographic proofs from AWS s2n-bignum. Submissions are evaluated by executing the synthesized tactics in HOL Light to verify correctness. It tracks LLM performance on a public leaderboard.","url":"https://github.com/kings-crown/s2n-bignum-bench","host":"github","status":"active","domain":"formal-methods","tags":["formal-methods","theorem-proving","cryptography","llm-eval","tactic-synthesis"],"images":[],"participation":{"howToSubmit":"Zip your answers CSV and submit by opening an issue using the leaderboard submission template at https://github.com/kings-crown/s2n-bignum-leaderboard/issues/new?template=leaderboard-submission.yml","submissionFormat":"CSV with columns problem_id, category, query, answer","computeDetails":"Each proof evaluation can use ~5 GB RAM. Evaluation runs on CPU.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"HOL Light proof checking (verdicts: OK, FAIL, CHEATING, ERROR)","requirements":["Python 3.9+","HOL Light","OCaml","opam","gmp"],"compute":"cpu"},"links":[{"label":"Paper","url":"https://arxiv.org/pdf/2603.14628"},{"label":"Leaderboard","url":"https://kings-crown.github.io/s2n-bignum-leaderboard/"},{"label":"Dataset on Hugging Face","url":"https://huggingface.co/datasets/kings-crown/s2n-bignum-bench"}],"organizer":{"name":"s2n-bignum-bench team","type":"academic"},"venue":{"type":"online"},"quickstart":"git clone https://github.com/kings-crown/s2n-bignum-bench\ncd s2n-bignum-bench\n./setup.sh arm 4\npython3 collect-problems.py toplevel-thms/ problems.json ml_files\npython3 retrieve-problem.py retrieve --outputdir workdir --csv problems.csv --csv-only","problems":[{"description":"Synthesize HOL Light tactic proofs for cryptographic theorems from AWS s2n-bignum.","metricUnit":"%","id":"tactic-synthesis","name":"HOL Light Tactic Synthesis","metricName":"accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"kings-crown","name":"s2n-bignum-bench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.142Z"},{"id":"agent-anvil-leaderboard","name":"Agent Anvil Leaderboard","tagline":"Compare final-answer-only checks with trace-aware assertions across agent tool-use safety scenarios.","description":"Agent Anvil is a public leaderboard for evaluating AI agents on tool-use safety scenarios. It compares traditional final-answer-only checks against trace-aware assertions to catch silent policy violations. Submissions are verified via GitHub Actions and artifact attestations to ensure provenance.","url":"https://github.com/agent-axiom/agent-anvil-leaderboard","host":"github","status":"active","domain":"llm-eval","tags":["agents","safety","evaluation","trace-analysis","tool-use"],"images":[],"participation":{"howToSubmit":"Run Agent Anvil in your repository, export the leaderboard submission JSON, and open a PR adding it under submissions/.","submissionFormat":"JSON file generated via `anvil leaderboard export`","computeDetails":"Run locally or in your own CI; the leaderboard repository only validates the submitted JSON and provenance.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"CI validates JSON schema, verifies GitHub run URLs, and checks artifact attestations","requirements":["Python","Agent Anvil","GitHub Actions"],"compute":"unknown"},"links":[{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/ifif/agent-anvil-leaderboard"},{"label":"Hugging Face Dataset","url":"https://huggingface.co/datasets/ifif/agent-anvil-leaderboard-data"},{"label":"Demo Agent Repository","url":"https://github.com/agent-axiom/agent-anvil-demo-agent"}],"organizer":{"name":"Agent Axiom","url":"https://github.com/agent-axiom","type":"company"},"quickstart":"uv run anvil paper reproduce\nuv run anvil leaderboard export docs/paper/results.json \\\n  --manifest experiments/paper.yaml \\\n  --out leaderboard_submission.json \\\n  --agent-name \"My Agent\"\nuv run anvil leaderboard validate leaderboard_submission.json","problems":[{"description":"A compact offline benchmark for comparing final-answer-only checks with trace-aware Agent Anvil assertions across tool-use safety scenarios.","metricUnit":"%","id":"agent_anvil_trace_eval_benchmark","name":"Agent Anvil Trace Eval Benchmark","metricName":"trace_aware_pass_rate","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Agent Anvil Demo Agent","description":"Demo agent run 26656805979","date":"2026-05-29","value":30,"contributorKind":"model","contributorUrl":"https://github.com/agent-axiom/agent-anvil-demo-agent","submissionUrls":["https://github.com/agent-axiom/agent-anvil-leaderboard/blob/main/submissions/agent-anvil-demo-agent.json"],"isBaseline":false,"isCurrentBest":true,"agent":{"model":"Agent Anvil Demo Agent","evidence":"Refresh agent-axiom-agent-anvil-demo-agent leaderboard submission","family":"axiom","confidence":"high","role":"subject"}}],"contested":false}],"agentStats":[],"repo":{"owner":"agent-axiom","name":"agent-anvil-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":1,"uniqueParticipants":0,"firstSubmission":"2026-05-29","lastSubmission":"2026-05-29","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.142Z"},{"id":"programbench","name":"ProgramBench","tagline":"Evaluate and track the performance of AI coding agents on software engineering tasks.","description":"ProgramBench is a benchmark for evaluating AI coding agents. Submissions are registered via pull requests containing execution manifests and per-instance scores, which are then verified and compiled into the public leaderboard.","url":"https://github.com/programbench/submissions","host":"github","status":"active","domain":"coding-agents","tags":["coding-agents","benchmark","software-engineering","swe-agent","llm-eval"],"images":[],"participation":{"howToSubmit":"Open a PR to main adding a folder under submissions/<submission_id>/ containing pointer.yaml, submission.yaml, and _stats/.","submissionFormat":"A folder containing pointer.yaml (fork URL + pinned commit SHA), submission.yaml (manifest), and _stats/ (score.json, cost.json, calls.json).","deadline":"rolling","verification":"CI verification using programbench submit verify (Tier-0 recomputes headline, Tier-1 re-runs eval in Docker)","requirements":["Python","Docker","uv"],"compute":"unknown"},"links":[{"label":"Leaderboard","url":"https://programbench.com/"},{"label":"Main Repository","url":"https://github.com/facebookresearch/ProgramBench"},{"label":"Submission Guide","url":"https://programbench.com/blog/submission-guide"}],"organizer":{"name":"Meta","url":"https://github.com/facebookresearch/ProgramBench","type":"company"},"quickstart":"git clone <your-submission-fork> && cd <fork>\nuvx programbench submit verify .","problems":[{"description":"Evaluation on the ProgramBench Mini dataset.","metricUnit":"%","id":"programbench-mini","name":"ProgramBench Mini","metricName":"mean score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GPT-5.5 (xhigh reasoning)","description":"resolved: 0.5% / near-resolved: 13.5%","date":"2026-05-08","value":69.8,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260508_mini-v2.2.6_gpt-5-5-xhigh","isCurrentBest":true,"agent":{"model":"GPT-5.5 (xhigh reasoning)","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (xhigh reasoning) (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"author"}},{"contributor":"GPT-5.5 (high reasoning)","description":"resolved: 0.5% / near-resolved: 5.0%","date":"2026-05-07","value":67,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260507_mini-v2.2.6_gpt-5-5-high","agent":{"model":"GPT-5.5 (high reasoning)","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (high reasoning) (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"author"}},{"contributor":"GPT-5.5","description":"resolved: 0.0% / near-resolved: 1.5%","date":"2026-05-07","value":56.8,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260507_mini-v2.2.6_gpt-5-5","agent":{"model":"GPT-5.5","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"author"}},{"contributor":"Claude Opus 4.6","description":"resolved: 0.0% / near-resolved: 2.5%","date":"2026-04-30","value":52.1,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260430_mini-v2.2.6_opus-4-6","agent":{"model":"Claude Opus 4.6","tool":"mini-SWE-agent","evidence":"Registers Claude Opus 4.6 (Anthropic) + mini-SWE-agent.","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"Claude Sonnet 4.6","description":"resolved: 0.0% / near-resolved: 1.5%","date":"2026-04-29","value":48.6,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_sonnet-4-6","agent":{"model":"Claude Sonnet 4.6","tool":"mini-SWE-agent","evidence":"Registers Claude Sonnet 4.6 (Anthropic) + mini-SWE-agent.","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"GPT-5.4","description":"resolved: 0.0% / near-resolved: 0.0%","date":"2026-04-29","value":38.7,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gpt-5-4","agent":{"model":"GPT-5.4","tool":"mini-SWE-agent","evidence":"Registers GPT-5.4 (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"author"}},{"contributor":"Gemini 3 Flash","description":"resolved: 0.0% / near-resolved: 0.0%","date":"2026-04-29","value":32.6,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gemini-3-flash","agent":{"model":"Gemini 3 Flash","tool":"mini-SWE-agent","evidence":"Registers Gemini 3 Flash (Google) + mini-SWE-agent.","family":"google","confidence":"high","role":"author"}},{"contributor":"Claude Haiku 4.5","description":"resolved: 0.0% / near-resolved: 0.0%","date":"2026-04-29","value":30.4,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_haiku-4-5","agent":{"model":"Claude Haiku 4.5","tool":"mini-SWE-agent","evidence":"Registers Claude Haiku 4.5 (Anthropic) + mini-SWE-agent.","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"GPT-5.4 mini","description":"resolved: 0.0% / near-resolved: 0.0%","date":"2026-04-29","value":17.1,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gpt-5-4-mini","agent":{"model":"GPT-5.4 mini","tool":"mini-SWE-agent","evidence":"Registers GPT-5.4 mini (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"author"}},{"contributor":"GPT-5 mini","description":"resolved: 0.0% / near-resolved: 0.0%","date":"2026-04-29","value":16.2,"contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gpt-5-mini","agent":{"model":"GPT-5 mini","tool":"mini-SWE-agent","evidence":"Registers GPT-5 mini (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"author"}}],"contested":true}],"agentStats":[{"family":"openai","records":6,"currentBests":1,"models":["GPT-5.5 (xhigh reasoning)","GPT-5.5 (high reasoning)","GPT-5.5","GPT-5.4","GPT-5.4 mini","GPT-5 mini"],"contributors":["GPT-5.5 (xhigh reasoning)","GPT-5.5 (high reasoning)","GPT-5.5","GPT-5.4","GPT-5.4 mini","GPT-5 mini"]},{"family":"anthropic","records":3,"currentBests":0,"models":["Claude Opus 4.6","Claude Sonnet 4.6","Claude Haiku 4.5"],"contributors":["Claude Opus 4.6","Claude Sonnet 4.6","Claude Haiku 4.5"]},{"family":"google","records":1,"currentBests":0,"models":["Gemini 3 Flash"],"contributors":["Gemini 3 Flash"]}],"repo":{"owner":"programbench","name":"submissions","branch":"main"},"participants":[],"stats":{"totalRecords":10,"uniqueParticipants":0,"firstSubmission":"2026-04-29","lastSubmission":"2026-05-08","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.143Z"},{"id":"chi-bench","name":"CHI-Bench Leaderboard","tagline":"Benchmark AI agents on healthcare administrative and clinical tasks.","description":"CHI-Bench evaluates AI agents on complex healthcare administrative and clinical workflows, including prior authorization and care management. Submissions are accepted via pull requests containing full audit packets with verifier evidence and trajectories. It is maintained by Actava AI to establish rigorous, reproducible agent evaluations in healthcare.","url":"https://github.com/actava-ai/leaderboard","host":"github","status":"active","domain":"llm-eval","tags":["healthcare","clinical-nlp","ai-agents","prior-authorization","care-management"],"images":[],"participation":{"howToSubmit":"Fork the repository, prepare your submission packet using the chi-bench CLI, and open a pull request adding the packet under benchmarks/chi-bench/submissions/<YYYY-MM-DD>-<slug>/.","submissionFormat":"A submission packet directory containing submission.json, results.csv, sub.yaml, provenance.json, README.md, and compressed trial trajectories.","verification":"Automated CI validation checks schema integrity, file sizes, and trial counts, followed by maintainer review.","requirements":["Python","uv","zstd","gh CLI"],"compute":"unknown"},"links":[{"label":"Live Leaderboard","url":"https://actava.ai/benchmarks/leaderboards"},{"label":"Submit Guide","url":"https://actava.ai/benchmarks/submit"},{"label":"Producer Repo","url":"https://github.com/actava-ai/chi-bench"}],"organizer":{"name":"Actava AI","url":"https://actava.ai/","type":"company"},"quickstart":"uv run cb submission prepare -f configs/submissions/<id>.yaml\npython scripts/submit.py /path/to/packet/2026-05-12-<slug>/","problems":[{"description":"Overall performance across all CHI-Bench tasks (75 tasks total).","metricUnit":"%","id":"chi-bench-overall","name":"Overall","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-5","date":"2026-07-27","value":54.7,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-5","date":"2026-07-24","value":37.3,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-05","value":37.3,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-4-8","date":"2026-05-28","value":33.3,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"My Team · claude-code · anthropic/claude-opus-4-6","date":"2026-05-12","value":28,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-6","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · moonshotai/kimi-k3","date":"2026-07-24","value":25.3,"contributorKind":"model","agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-sol","date":"2026-07-24","value":25.3,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-sol","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-fable-5","date":"2026-07-22","value":24,"contributorKind":"model","agent":{"model":"anthropic/claude-fable-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"cuilinke · hermes · MedGuard","date":"2026-07-06","value":22.7,"contributorKind":"model","agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-sonnet-5","date":"2026-07-06","value":20,"contributorKind":"model","agent":{"model":"anthropic/claude-sonnet-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · z-ai/glm-5.2","date":"2026-07-06","value":18.7,"contributorKind":"model","agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-02","value":18.7,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-luna","date":"2026-07-24","value":13.3,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-luna","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-terra","date":"2026-07-24","value":13.3,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-terra","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · thinkingmachines/Inkling:peft:262144","date":"2026-07-24","value":8,"contributorKind":"model","agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","date":"2026-07-24","value":0,"contributorKind":"model","agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Performance on Provider Prior Authorization tasks (25 tasks total).","metricUnit":"%","id":"chi-bench-pa-provider","name":"Provider Prior Authorization","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-5","date":"2026-07-27","value":72,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"MedArise · codex · openai/gpt-5.5","date":"2026-06-05","value":68,"contributorKind":"model","agent":{"model":"openai/gpt-5.5","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-05","value":40,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-02","value":40,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-sol","date":"2026-07-24","value":36,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-sol","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-4-8","date":"2026-05-28","value":32,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · moonshotai/kimi-k3","date":"2026-07-24","value":28,"contributorKind":"model","agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-fable-5","date":"2026-07-22","value":24,"contributorKind":"model","agent":{"model":"anthropic/claude-fable-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-sonnet-5","date":"2026-07-06","value":24,"contributorKind":"model","agent":{"model":"anthropic/claude-sonnet-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-5","date":"2026-07-24","value":20,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-luna","date":"2026-07-24","value":20,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-luna","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · z-ai/glm-5.2","date":"2026-07-06","value":20,"contributorKind":"model","agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"My Team · claude-code · anthropic/claude-opus-4-6","date":"2026-05-12","value":20,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-6","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-terra","date":"2026-07-24","value":12,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-terra","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · thinkingmachines/Inkling:peft:262144","date":"2026-07-24","value":4,"contributorKind":"model","agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cuilinke · hermes · MedGuard","date":"2026-07-06","value":4,"contributorKind":"model","agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","date":"2026-07-24","value":0,"contributorKind":"model","agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Performance on Payer Utilization Management tasks (25 tasks total).","metricUnit":"%","id":"chi-bench-pa-um","name":"Payer Utilization Management","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-5","date":"2026-07-27","value":36,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"My Team · claude-code · anthropic/claude-opus-4-6","date":"2026-05-12","value":36,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-6","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-5","date":"2026-07-24","value":32,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · moonshotai/kimi-k3","date":"2026-07-24","value":32,"contributorKind":"model","agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · z-ai/glm-5.2","date":"2026-07-06","value":32,"contributorKind":"model","agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-sol","date":"2026-07-24","value":28,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-sol","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-4-8","date":"2026-05-28","value":28,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-fable-5","date":"2026-07-22","value":24,"contributorKind":"model","agent":{"model":"anthropic/claude-fable-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-sonnet-5","date":"2026-07-06","value":24,"contributorKind":"model","agent":{"model":"anthropic/claude-sonnet-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-terra","date":"2026-07-24","value":20,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-terra","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · thinkingmachines/Inkling:peft:262144","date":"2026-07-24","value":16,"contributorKind":"model","agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-luna","date":"2026-07-24","value":16,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-luna","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-05","value":16,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-02","value":16,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"cuilinke · hermes · MedGuard","date":"2026-07-06","value":4,"contributorKind":"model","agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","date":"2026-07-24","value":0,"contributorKind":"model","agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Performance on Care Management tasks (25 tasks total).","metricUnit":"%","id":"chi-bench-cm","name":"Care Management","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Actava · claude-code · anthropic/claude-opus-5","date":"2026-07-24","value":60,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"cuilinke · hermes · MedGuard","date":"2026-07-06","value":60,"contributorKind":"model","agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-5","date":"2026-07-27","value":56,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-05","value":56,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-opus-4-8","date":"2026-05-28","value":40,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"My Team · claude-code · anthropic/claude-opus-4-6","date":"2026-05-12","value":28,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-6","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-fable-5","date":"2026-07-22","value":24,"contributorKind":"model","agent":{"model":"anthropic/claude-fable-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · moonshotai/kimi-k3","date":"2026-07-24","value":16,"contributorKind":"model","agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-sol","date":"2026-07-24","value":12,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-sol","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · claude-code · anthropic/claude-sonnet-5","date":"2026-07-06","value":12,"contributorKind":"model","agent":{"model":"anthropic/claude-sonnet-5","tool":"Claude Code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-terra","date":"2026-07-24","value":8,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-terra","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · thinkingmachines/Inkling:peft:262144","date":"2026-07-24","value":4,"contributorKind":"model","agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Actava · codex · openai/gpt-5.6-luna","date":"2026-07-24","value":4,"contributorKind":"model","agent":{"model":"openai/gpt-5.6-luna","tool":"Codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · z-ai/glm-5.2","date":"2026-07-06","value":4,"contributorKind":"model","agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","date":"2026-06-02","value":0,"contributorKind":"model","agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Actava · openai-agents · nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","date":"2026-07-24","value":0,"contributorKind":"model","agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"actava-ai","name":"leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":65,"uniqueParticipants":0,"firstSubmission":"2026-05-12","lastSubmission":"2026-07-27","recordsLast30d":0,"recordsLast90d":36},"lastUpdated":"2026-10-13T12:55:04.143Z"},{"id":"geo-bench-2-leaderboard","name":"GEO-Bench Leaderboard","tagline":"Benchmark geospatial foundation models on diverse Earth observation tasks.","description":"A leaderboard tracking the performance of geospatial foundation models using the GEO-Bench benchmarking framework. It evaluates models on tasks like classification and regression using metrics like accuracy, F1 score, Jaccard index, and RMSE. Submissions are made via pull requests with standardized evaluation results.","url":"https://github.com/The-AI-Alliance/GEO-Bench-2-Leaderboard","host":"github","status":"active","domain":"ml-research","tags":["geospatial","earth-observation","foundation-models","remote-sensing","machine-learning"],"images":[],"participation":{"howToSubmit":"Fork the repository, create a new folder under `new_submission/` with `results_and_parameters.csv` and `additional_info.json`, and open a pull request to main.","submissionFormat":"CSV file (`results_and_parameters.csv`) containing backbone, dataset, metric, hyperparameters, and test metrics, plus a JSON file (`additional_info.json`) with submission details.","verification":"CI and manual review of PR","requirements":["Python 3.10+","TerraTorch","MLflow","Optuna"],"compute":"unknown"},"links":[{"label":"Hugging Face Leaderboard","url":"https://huggingface.co/spaces/aialliance/GEO-Bench-2-Leaderboard"},{"label":"TerraTorch-Iterate GitHub","url":"https://github.com/IBM/terratorch-iterate"},{"label":"TerraTorch GitHub","url":"https://github.com/IBM/terratorch"}],"organizer":{"name":"The AI Alliance","url":"https://thealliance.ai/","type":"community"},"quickstart":"# Run benchmark experiments on existing models\nterratorch iterate --hpo --repeat --config <config-file>\n\n# Extract results and parameters file for submission\nterratorch iterate --summarize --config <summarize-config-file>","problems":[{"id":"overall-accuracy","name":"Overall Accuracy Track","metricName":"Overall Accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"id":"multilabel-f1","name":"Multilabel F1 Score Track","metricName":"Multilabel F1 Score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"id":"multiclass-jaccard","name":"Multiclass Jaccard Index Track","metricName":"Multiclass Jaccard Index","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"id":"biomassters-rmse","name":"Biomassters RMSE Track","metricName":"RMSE","metricDirection":"minimize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"The-AI-Alliance","name":"GEO-Bench-2-Leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.144Z"},{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","tagline":"Speedrun training NanoGPT to 3.28 validation loss on FineWeb using Ascend NPUs.","description":"This repository tracks NPU reproductions of the modded-nanogpt speedrun leaderboard on 16×Ascend 910C NPUs. It ports, reviews, and benchmarks upstream GPU records to evaluate NPU performance and optimization techniques.","url":"https://github.com/zhaoyiyidan/modded-nanogpt-npu","host":"github","status":"active","domain":"systems-perf","tags":["nanogpt","npu","ascend","speedrun","llm-training","pytorch"],"images":["https://raw.githubusercontent.com/zhaoyiyidan/modded-nanogpt-npu/master/img/algo_optimizer.png","https://raw.githubusercontent.com/zhaoyiyidan/modded-nanogpt-npu/master/img/nanogpt_speedrun51.png","https://raw.githubusercontent.com/zhaoyiyidan/modded-nanogpt-npu/master/img/nanogpt_speedrun52.png"],"participation":{"howToSubmit":"Open a PR to port and benchmark an upstream GPU record on the 16x Ascend 910C NPU runtime.","submissionFormat":"NPU source code under records/track_1_short_npu/<record>/source/ and successful run logs under records/track_1_short_npu/<record>/logs/","computeDetails":"16×Ascend 910C NPU","requirements":["Python 3.10+","PyTorch","Ascend NPU","MindSpeed","Docker"],"compute":"cluster"},"links":[{"label":"Upstream Repository","url":"https://github.com/KellerJordan/modded-nanogpt"},{"label":"Muon Optimizer Writeup","url":"https://kellerjordan.github.io/posts/muon/"}],"organizer":{"name":"zhaoyiyidan","type":"individual"},"quickstart":"git clone https://github.com/zhaoyiyidan/modded-nanogpt-npu.git && cd modded-nanogpt-npu\ntorchrun --standalone --nproc_per_node=16 train_gpt.py","problems":[{"description":"Train a language model to <=3.28 validation loss on FineWeb using 16x Ascend 910C NPUs.","metricUnit":"minutes","id":"track-1-short-npu","name":"Track 1 Short (16x Ascend 910C NPU)","metricName":"time","metricDirection":"minimize","baseline":104.72,"isOpen":false,"records":[{"contributor":"record_001_llmc_baseline","description":"llm.c baseline; baseline/reference port, no specific speed-up trick.","value":104.72,"contributorKind":"method","isBaseline":true},{"contributor":"record_002_adamw_rotary_lr","description":"Tuned learning rate and rotary embeddings.","value":61.6,"contributorKind":"method"},{"contributor":"record_005_modern_arch","description":"Pad embeddings, ReLU2, zero-init projections, QK-norm.","value":42.18,"contributorKind":"method"},{"contributor":"record_004_muon_improvements","description":"Muon optimizer improvements.","value":40.43,"contributorKind":"method"},{"contributor":"record_006_distributed_muon","description":"Distributed Muon overhead.","value":40.33,"contributorKind":"method"},{"contributor":"record_007_pytorch25","description":"PyTorch 2.5 runtime upgrade.","value":40.12,"contributorKind":"method"},{"contributor":"record_008_untie_embed","description":"Untied token embedding and LM head.","value":36.6,"contributorKind":"method"},{"contributor":"record_010_cast_bf16","description":"Cast activations to bfloat16.","value":33.96,"contributorKind":"method"},{"contributor":"record_009_shortcuts_tweaks","description":"Value/embedding skip connections, momentum warmup, logit softcap.","value":31.83,"contributorKind":"method"},{"contributor":"record_021_batch_size","description":"Reduced batch size.","value":29.56,"contributorKind":"method"},{"contributor":"record_017_sparsify_embeds","description":"Sparsified value embeddings, improved rotary embeddings, dropped one attention layer.","value":19.72,"contributorKind":"method"},{"contributor":"record_027_triton_muon","description":"Transposed one MLP matrix and added a symmetric matmul kernel.","value":12.58,"contributorKind":"method"},{"contributor":"record_030_skip_mlp_blocks","description":"Dropped the first MLP layer.","value":11.65,"contributorKind":"method"},{"contributor":"record_032_vect_sigmoid_bfloat16","description":"Optimized distributed training, improved skip-connection gating, enhanced bfloat16 usage.","value":11.06,"contributorKind":"method"},{"contributor":"record_034_smear","description":"Smeared token embeddings one position forward.","value":11.05,"contributorKind":"method"},{"contributor":"record_033_async_data_attn_final_window","description":"Asynchronous data fetch/indexing and final-layer validation attention window extension.","value":10.92,"contributorKind":"method"},{"contributor":"record_035_drop_attn","description":"Dropped first attention layer, extended long validation windows, updated schedule.","value":10.77,"contributorKind":"method"},{"contributor":"record_038_polar_express","description":"Polar Express replacement for Newton-Schulz.","value":10.76,"contributorKind":"method"},{"contributor":"record_039_custom_batching","description":"Updated Adam params every other step and reduced batch size.","value":10.53,"contributorKind":"method"},{"contributor":"record_040_backout","description":"Backout changes, hyperparameter tuning, lambda padding optimization.","value":10.16,"contributorKind":"method"},{"contributor":"record_042_fix_muon_lr","description":"Updated NorMuon learning rate and step logic.","value":9.98,"contributorKind":"method"},{"contributor":"record_041_normuon","description":"NorMuon optimizer.","value":9.69,"contributorKind":"method"},{"contributor":"record_045_refine_skip","description":"Refined skip architecture, updated exponential decay init.","value":9.65,"contributorKind":"method"},{"contributor":"record_046_batch_size_schedule","description":"Batch size schedule.","value":9.57,"contributorKind":"method"},{"contributor":"record_048_normuon_optims_and_fixes","description":"Sped up Muon, pre-multiplied lambda, reshaped matrices, updated lr and NorMuon axis.","value":9.56,"contributorKind":"method"},{"contributor":"record_049_partial_key_offset","description":"Partial Key Offset.","value":9.48,"contributorKind":"method"},{"contributor":"record_050_cautious_wd_adam","description":"Extended Cautious Weight Decay to Adam parameters.","value":9.38,"contributorKind":"method"},{"contributor":"record_037_bf16_ce","description":"Computed cross entropy in BF16 during training.","value":9.34,"contributorKind":"method"},{"contributor":"record_047_sa_lambda_on_weights","description":"Multiplied attention lambda with weight instead of data, fixed warmup.","value":9.04,"contributorKind":"method"},{"contributor":"record_050_cautious_wd_adam_npu_optimized","description":"NPU-optimized Cautious Weight Decay on Adam: fused RMSNorm/rotary, buffer reuse, bfloat16 weights, a 2070-step schedule, and reduced validation/metrics overhead.","value":5.78,"contributorKind":"method","isCurrentBest":true}],"contested":true},{"description":"Train a language model to <=3.28 validation loss on FineWeb using 8x NVIDIA H100 GPUs.","metricUnit":"minutes","id":"track-1-short-gpu","name":"Track 1 Short (8x NVIDIA H100 GPU)","metricName":"time","metricDirection":"minimize","baseline":45,"isOpen":false,"records":[{"contributor":"karpathy, llm.c contributors","description":"llm.c baseline","date":"2024-05-28","value":45,"contributorKind":"team","isBaseline":true},{"contributor":"kellerjordan0","description":"Tuned learning rate & rotary embeddings","date":"2024-06-06","value":31.4,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"kellerjordan0, jxbz","description":"Introduced the Muon optimizer","date":"2024-10-04","value":24.9,"contributorKind":"team"},{"contributor":"kellerjordan0, bozavlado","description":"Muon improvements","date":"2024-10-11","value":22.3,"contributorKind":"team"},{"contributor":"Grad62304977, kellerjordan0","description":"Pad embeddings, ReLU², zero-init projections, QK-norm","date":"2024-10-14","value":15.2,"contributorKind":"team"},{"contributor":"kellerjordan0","description":"Distributed the overhead of Muon","date":"2024-10-18","value":13.1,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"kellerjordan0","description":"Upgraded PyTorch 2.5.0","date":"2024-10-18","value":12,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"Grad62304977, kellerjordan0","description":"Untied embedding and head","date":"2024-11-03","value":10.8,"contributorKind":"team"},{"contributor":"Grad62304977, kellerjordan0","description":"Value and embedding skip connections, momentum warmup, logit softcap","date":"2024-11-06","value":8.2,"contributorKind":"team"},{"contributor":"kellerjordan0","description":"Bfloat16 activations","date":"2024-11-08","value":7.8,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"brendanh0gan","description":"U-net pattern skip connections & double lr","date":"2024-11-10","value":7.2,"contributorKind":"person","contributorUrl":"https://github.com/brendanh0gan"},{"contributor":"KoszarskyB","description":"1024-ctx dense causal attention → 64K-ctx FlexAttention","date":"2024-11-19","value":5.03,"contributorKind":"person","contributorUrl":"https://github.com/KoszarskyB"},{"contributor":"fernbear.bsky.social","description":"Attention window warmup","date":"2024-11-24","value":4.66,"contributorKind":"person"},{"contributor":"KoszarskyB","description":"Value Embeddings","date":"2024-12-04","value":4.41,"contributorKind":"person","contributorUrl":"https://github.com/KoszarskyB"},{"contributor":"leloykun, YouJiacheng","description":"U-net pattern value embeddings, assorted code optimizations","date":"2024-12-08","value":3.95,"contributorKind":"team"},{"contributor":"YouJiacheng","description":"Split value embeddings, block sliding window, separate block mask","date":"2024-12-10","value":3.8,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng"},{"contributor":"YouJiacheng","description":"Sparsify value embeddings, improve rotary embeddings, drop an attn layer","date":"2024-12-17","value":3.57,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng"},{"contributor":"KoszarskyB","description":"Lower logit softcap from 30 to 15","date":"2025-01-04","value":3.4,"contributorKind":"person","contributorUrl":"https://github.com/KoszarskyB"},{"contributor":"YouJiacheng","description":"FP8 head, offset logits, lr decay to 0.1 instead of 0.0","date":"2025-01-13","value":3.142,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng"},{"contributor":"leloykun, fernbear.bsky.social, YouJiacheng, brendanh0gan, scottjmaddox, Grad62304977","description":"Merged QKV weights, long-short attention, attention scale, lower Adam epsilon, batched Muon","date":"2025-01-16","value":2.992,"contributorKind":"team"},{"contributor":"leloykun","description":"Reduced batch size","date":"2025-01-26","value":2.933,"contributorKind":"person","contributorUrl":"https://github.com/leloykun"},{"contributor":"not a new record, just re-timing #21","description":"21st record with new timing","date":"2025-02-01","value":2.997,"contributorKind":"method"},{"contributor":"not a new record, just re-timing #21","description":"21st record with latest torch","date":"2025-05-24","value":3.014,"contributorKind":"method"},{"contributor":"KonstantinWilleke, alexrgilbert, adricarda, tuttyfrutyee, vdlad","description":"Faster gradient all-reduce","date":"2025-05-24","value":2.99,"contributorKind":"team"},{"contributor":"ryanyang0","description":"Overlap computation and gradient communication","date":"2025-05-25","value":2.979,"contributorKind":"person","contributorUrl":"https://github.com/ryanyang0"},{"contributor":"vagrawal","description":"Replace gradient all_reduce with reduce_scatter","date":"2025-05-30","value":2.966,"contributorKind":"person","contributorUrl":"https://github.com/vagrawal"},{"contributor":"kellerjordan0","description":"Upgrade PyTorch to 2.9.0.dev20250713+cu126","date":"2025-07-13","value":2.896,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"classiclarryd","description":"Align training batch starts with EoS, increase cooldown frac to .45","date":"2025-07-13","value":2.863,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"byronxu99","description":"Transpose one of the MLP matrices + add Triton kernel for symmetric matmul","date":"2025-07-18","value":2.817,"contributorKind":"person","contributorUrl":"https://github.com/byronxu99"},{"contributor":"classiclarryd","description":"Sparse attention gate","date":"2025-08-23","value":2.812,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"varunneal","description":"Flash Attention 3, 2048 max_doc_len, update ws schedule","date":"2025-09-03","value":2.731,"contributorKind":"person","contributorUrl":"https://github.com/varunneal"},{"contributor":"EmelyanenkoK","description":"Drop first MLP layer","date":"2025-09-05","value":2.717,"contributorKind":"person","contributorUrl":"https://github.com/EmelyanenkoK"},{"contributor":"classiclarryd","description":"Dynamically incorporate YaRN during training and validation","date":"2025-09-10","value":2.656,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"bernard24","description":"Optimize distributed training, improve skip connection gating, and enhance bfloat16 usage","date":"2025-09-11","value":2.625,"contributorKind":"person","contributorUrl":"https://github.com/bernard24"},{"contributor":"classiclarryd","description":"Asynchronously fetch and index data batches, extend final layer attention window for validation","date":"2025-09-15","value":2.565,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"classiclarryd","description":"Smear token embeddings 1 position forward","date":"2025-09-18","value":2.547,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"classiclarryd","description":"Drop first attn layer, extend all long windows for validation, update schedule","date":"2025-09-21","value":2.527,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"classiclarryd","description":"MuonCustomSizing, perform mlp and attn reduce scatter in shared call","date":"2025-09-23","value":2.495,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"Gusarich","description":"Compute cross entropy in BF16 during training","date":"2025-09-27","value":2.483,"contributorKind":"person","contributorUrl":"https://github.com/Gusarich"},{"contributor":"varunneal","description":"Polar Express, replacement for Newton-Schulz","date":"2025-09-29","value":2.476,"contributorKind":"person","contributorUrl":"https://github.com/varunneal"},{"contributor":"classiclarryd","description":"Only update Adam params every other step, reduce batch size","date":"2025-09-30","value":2.447,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"classiclarryd","description":"Backout, misc hyperparameter tuning, optimize lambda padding","date":"2025-10-04","value":2.358,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"li_zichong","description":"NorMuon","date":"2025-10-24","value":2.345,"contributorKind":"person","contributorUrl":"https://github.com/li_zichong"},{"contributor":"varunneal","description":"Update NorMuon LR, Step Logic","date":"2025-10-27","value":2.313,"contributorKind":"person","contributorUrl":"https://github.com/varunneal"},{"contributor":"varunneal","description":"Cautious Weight Decay w/ schedule","date":"2025-11-10","value":2.284,"contributorKind":"person","contributorUrl":"https://github.com/varunneal"},{"contributor":"akash5474","description":"Backward hooks on Adam, Profiling 101","date":"2025-11-16","value":2.269,"contributorKind":"person","contributorUrl":"https://github.com/akash5474"},{"contributor":"classiclarryd","description":"Refine skip arch, update exponential decay init","date":"2025-11-18","value":2.248,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd"},{"contributor":"varunneal","description":"Batch size schedule","date":"2025-11-29","value":2.203,"contributorKind":"person","contributorUrl":"https://github.com/varunneal"},{"contributor":"roeeshenberg","description":"Multiply attn lambda with weight instead of data, fix warmup","date":"2025-12-10","value":2.193,"contributorKind":"person","contributorUrl":"https://github.com/roeeshenberg"},{"contributor":"ChrisJMcCormick","description":"Speed up Muon, additional pre-multiply lambda, reshape matrices, update lr, update NorMuon axis","date":"2025-12-11","value":2.17,"contributorKind":"person","contributorUrl":"https://github.com/ChrisJMcCormick"},{"contributor":"classiclarryd","description":"Partial Key Offset","date":"2025-12-14","value":2.146,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","isCurrentBest":true}],"contested":true},{"description":"Train a language model to <=2.92 validation loss on FineWeb using 8x NVIDIA H100 GPUs.","metricUnit":"minutes","id":"track-2-medium-gpu","name":"Track 2 Medium (8x NVIDIA H100 GPU)","metricName":"time","metricDirection":"minimize","baseline":348,"isOpen":false,"records":[{"contributor":"karpathy, llm.c contributors","description":"llm.c baseline (350M parameters)","date":"2024-05-28","value":348,"contributorKind":"team","isBaseline":true},{"contributor":"kellerjordan0","description":"Initial record based on scaling up the GPT-2 small track speedrun","date":"2025-01-18","value":29.3,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"kellerjordan0","description":"Added standard weight decay","date":"2025-02-08","value":28.1,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0"},{"contributor":"leloykun","description":"Tuned Muon Newton-Schulz coefficients","date":"2025-02-14","value":27.7,"contributorKind":"person","contributorUrl":"https://github.com/leloykun"},{"contributor":"YouJiacheng","description":"Increased learning rate cooldown phase duration","date":"2025-03-06","value":27.2,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng"},{"contributor":"YouJiacheng","description":"2x MLP wd, qkv norm, all_reduce/opt.step() overlap, optimized skip pattern","date":"2025-03-25","value":25.95,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng"},{"contributor":"YouJiacheng","description":"Remove FP8 head; ISRU logits softcap; New sharded mixed precision Muon; merge weights","date":"2025-04-16","value":25.29,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng"},{"contributor":"jadenj3o","description":"Cubic sliding window size schedule, 2× max window size","date":"2025-04-22","value":24.5,"contributorKind":"person","contributorUrl":"https://github.com/jadenj3o"},{"contributor":"snimu","description":"Add two value embeddings","date":"2025-08-28","value":24.12,"contributorKind":"person","contributorUrl":"https://github.com/snimu"},{"contributor":"snimu","description":"Second input embedding","date":"2025-09-11","value":24.07,"contributorKind":"person","contributorUrl":"https://github.com/snimu","isCurrentBest":true}],"contested":true}],"agentStats":[],"repo":{"owner":"zhaoyiyidan","name":"modded-nanogpt-npu","branch":"master"},"participants":[{"name":"classiclarryd","url":"https://github.com/classiclarryd","submissions":11,"problems":["track-1-short-gpu"],"lastActive":"2025-12-14","bestRank":1},{"name":"snimu","url":"https://github.com/snimu","submissions":2,"problems":["track-2-medium-gpu"],"lastActive":"2025-09-11","bestRank":1},{"name":"kellerjordan0","url":"https://github.com/kellerjordan0","submissions":12,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-07-13"},{"name":"YouJiacheng","url":"https://github.com/YouJiacheng","submissions":7,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-04-16"},{"name":"varunneal","url":"https://github.com/varunneal","submissions":5,"problems":["track-1-short-gpu"],"lastActive":"2025-11-29"},{"name":"Grad62304977","url":"https://github.com/Grad62304977","submissions":3,"problems":["track-1-short-gpu"],"lastActive":"2024-11-06"},{"name":"KoszarskyB","url":"https://github.com/KoszarskyB","submissions":3,"problems":["track-1-short-gpu"],"lastActive":"2025-01-04"},{"name":"leloykun","url":"https://github.com/leloykun","submissions":3,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-02-14"},{"name":"karpathy, llm.c contributors","submissions":2,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2024-05-28"},{"name":"jxbz","url":"https://github.com/jxbz","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-10-04"},{"name":"bozavlado","url":"https://github.com/bozavlado","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-10-11"},{"name":"brendanh0gan","url":"https://github.com/brendanh0gan","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-11-10"},{"name":"fernbear.bsky.social","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-11-24"},{"name":"leloykun, fernbear.bsky.social, YouJiacheng, brendanh0gan, scottjmaddox, Grad62304977","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-01-16"},{"name":"KonstantinWilleke","url":"https://github.com/KonstantinWilleke","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"alexrgilbert","url":"https://github.com/alexrgilbert","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"adricarda","url":"https://github.com/adricarda","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"tuttyfrutyee","url":"https://github.com/tuttyfrutyee","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"vdlad","url":"https://github.com/vdlad","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"ryanyang0","url":"https://github.com/ryanyang0","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-25"},{"name":"vagrawal","url":"https://github.com/vagrawal","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-30"},{"name":"byronxu99","url":"https://github.com/byronxu99","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-07-18"},{"name":"EmelyanenkoK","url":"https://github.com/EmelyanenkoK","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-05"},{"name":"bernard24","url":"https://github.com/bernard24","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-11"},{"name":"Gusarich","url":"https://github.com/Gusarich","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-27"},{"name":"li_zichong","url":"https://github.com/li_zichong","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-10-24"},{"name":"akash5474","url":"https://github.com/akash5474","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-11-16"},{"name":"roeeshenberg","url":"https://github.com/roeeshenberg","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-12-10"},{"name":"ChrisJMcCormick","url":"https://github.com/ChrisJMcCormick","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-12-11"},{"name":"jadenj3o","url":"https://github.com/jadenj3o","submissions":1,"problems":["track-2-medium-gpu"],"lastActive":"2025-04-22"}],"stats":{"totalRecords":91,"uniqueParticipants":30,"firstSubmission":"2024-05-28","lastSubmission":"2025-12-14","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.144Z"},{"id":"apex-humanoid-parkour","name":"Humanoid Parkour","tagline":"Optimize an ONNX policy to drive a Unitree G1 humanoid through a 51m parkour course.","description":"An Apex competition (Bittensor Subnet 1) where participants submit an ONNX policy to control a Unitree G1 humanoid robot. The robot must navigate a 51m parkour course with static geometry but randomized friction and wind. Submissions are evaluated over 24 instances and scored based on progress or completion speed.","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour","host":"github","status":"active","domain":"robotics","tags":["humanoid","robotics","reinforcement-learning","mujoco","onnx","bittensor"],"images":["https://github.com/user-attachments/assets/17b9f4ca-39ef-4d15-a216-b3eee1888a43"],"participation":{"howToSubmit":"Submit your ONNX policy to the Bittensor Subnet 1 (Apex) miner network.","submissionFormat":"ONNX graph (<= 15 MB) matching the interface: obs[104] + state_in[256] -> action[12] + state_out[256]","computeDetails":"Evaluation runs in Docker containers limited to 1 CPU and 1.5 GB memory.","deadline":"rolling","prizes":"Bittensor TAO rewards","requirements":["Python","ONNX","MuJoCo","Docker"],"compute":"cpu"},"links":[{"label":"GitHub Repository","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour"},{"label":"Unitree RL Gym","url":"https://github.com/unitreerobotics/unitree_rl_gym"}],"organizer":{"name":"Macrocosm OS","url":"https://github.com/macrocosm-os","type":"community"},"quickstart":"docker build -f referee/Dockerfile -t hp-referee .\ndocker build -f player/Dockerfile  -t hp-player  .\ndocker network create hpnet\ndocker run -d --name hp-p --network hpnet -v \"$PWD/baseline/baseline.onnx:/app/submission.onnx:ro\" hp-player","problems":[{"description":"Drive a Unitree G1 humanoid through a 51 m parkour course with randomized friction and wind.","id":"humanoid-parkour-course","name":"Humanoid Parkour Course","metricName":"score","metricDirection":"maximize","baseline":0.21,"isOpen":true,"records":[{"contributor":"Unitree Stock G1 Walker","description":"Unitree's stock G1 walking policy (deploy/pre_train/g1/motion.pt) wrapped to speak the interface.","value":0.21,"contributorKind":"model","isBaseline":true,"isCurrentBest":true,"agent":{"model":"Unitree Stock G1 Walker","evidence":"Unitree Stock G1 Walker","family":"other-ai","confidence":"high","role":"subject"}}],"contested":false}],"agentStats":[],"repo":{"owner":"macrocosm-os","name":"apex-competition-humanoid-parkour","branch":"main"},"participants":[],"stats":{"totalRecords":1,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.145Z"},{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","tagline":"Evaluate coding agents on SWE-bench Pro, a dataset of 731 real-world software engineering tasks.","description":"SWE-bench Pro is a benchmark for evaluating coding agents on real-world software engineering tasks. Agents are evaluated by running the project's test suite in Docker containers to verify bug fixes and prevent regressions. It is run by the RDI Foundation to track state-of-the-art coding agents.","url":"https://github.com/RDI-Foundation/swe-bench-leaderboard","host":"github","status":"active","domain":"coding-agents","tags":["swe-bench","coding-agents","software-engineering","agentbeats"],"images":[],"participation":{"howToSubmit":"Fork the repository, edit scenario.json5 with your agentbeats ID, add API keys as secrets, and open a PR, or submit directly via agentbeats.dev.","submissionFormat":"scenario.json5 configuration or agentbeats.dev UI submission","computeDetails":"Evaluation runs in Docker containers to execute test suites.","deadline":"rolling","prizes":"recognition only","eligibility":"open to everyone","cost":"free","verification":"CI runs the agent and executes the project's test suite in Docker","requirements":["Docker","Python","Amber CLI"],"compute":"unknown"},"links":[{"label":"SWE-bench Pro Dataset","url":"https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"},{"label":"AgentBeats","url":"https://agentbeats.dev/"},{"label":"Amber CLI","url":"https://github.com/rdi-foundation/amber-cli"},{"label":"SWE-bench Green Agent","url":"https://github.com/rdi-foundation/swe-bench-green-agent"},{"label":"SWE-bench Purple Agent","url":"https://github.com/rdi-foundation/swe-bench-purple-agent"}],"organizer":{"name":"RDI Foundation","url":"https://github.com/RDI-Foundation","type":"foundation"},"quickstart":"amber compile scenario.json5 --docker-compose amber-out\ncp amber-out/env.example amber-out/.env\nAMBER_CONFIG_CODING_AGENT__MODEL_NAME=gpt-4o AMBER_CONFIG_CODING_AGENT__OPENAI_API_KEY=sk-xxx docker compose -f amber-out/compose.yaml --env-file amber-out/.env up","problems":[{"description":"Evaluating coding agents on 731 real-world software engineering tasks from SWE-bench Pro.","metricUnit":"%","id":"swe-bench-pro","name":"SWE-bench Pro","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"SWE-bench Purple Agent (deepseek-chat)","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","date":"2026-04-16","value":14.29,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9602-bdf5-7b71-8185-1ba4e00fa994.json"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"SWE-bench Purple Agent (deepseek-v3.2)","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","date":"2026-04-13","value":8.57,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d86f7-8639-7853-bbd1-b045cc625e16.json"],"isBaseline":true,"isCurrentBest":false,"agent":{"model":"deepseek-v3.2","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"SWE-bench Purple Agent (deepseek-chat)","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","date":"2026-04-15","value":8.57,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9025-c7b0-7240-b0e0-069d053c12c7.json"],"isBaseline":true,"isCurrentBest":false,"agent":{"model":"deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"SWE-bench Purple Agent (deepseek-chat)","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","date":"2026-04-16","value":2.86,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9480-9365-7273-b12a-bd76d7d604b4.json"],"isBaseline":true,"isCurrentBest":false,"agent":{"model":"deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"SWE-bench Purple Agent (deepseek-v3.2)","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","date":"2026-04-13","value":0,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d8691-0a8c-7a32-a5e1-1b72cf211f3a.json"],"isBaseline":true,"isCurrentBest":false,"agent":{"model":"deepseek-v3.2","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"AgentX-Spring0123","description":"Purple agent using official A2A Python server template.","date":"2026-04-18","value":0,"contributorKind":"team","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019da114-0d54-7310-b6be-58c57d201617.json"],"isBaseline":false,"isCurrentBest":false,"agent":{"tool":"AgentX","evidence":"Purple agent using official A2A Python server template.","family":"other-ai","confidence":"medium","role":"author"}}],"contested":false}],"agentStats":[{"family":"other-ai","records":1,"currentBests":0,"models":[],"contributors":["AgentX-Spring0123"]}],"repo":{"owner":"RDI-Foundation","name":"swe-bench-leaderboard","branch":"main"},"participants":[{"name":"AgentX-Spring0123","submissions":1,"problems":["swe-bench-pro"],"lastActive":"2026-04-18"}],"stats":{"totalRecords":6,"uniqueParticipants":1,"firstSubmission":"2026-04-13","lastSubmission":"2026-04-18","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.145Z"},{"id":"mle-bench","name":"MLE-bench","tagline":"Evaluating machine learning agents on machine learning engineering tasks across 75 Kaggle competitions.","description":"MLE-bench is a benchmark designed to evaluate the machine learning engineering capabilities of AI agents. It consists of 75 Kaggle competitions with prepared training/test splits and deterministic grading scripts. Agents are evaluated on their ability to train models, optimize pipelines, and produce competitive submissions.","url":"https://github.com/openai/mle-bench","host":"github","status":"active","domain":"coding-agents","tags":["ml-engineering","agents","benchmark","kaggle","llm-eval"],"images":[],"participation":{"howToSubmit":"Open a Pull Request to the main branch adding grading reports under the runs/ folder and updating the README.md leaderboard table.","submissionFormat":"Grading reports in JSON format under the runs/ folder, and updating the README.md leaderboard table.","computeDetails":"Canonical setup recommends 36 vCPUs, 440GB RAM, and one 24GB A10 GPU per agent run, with a 24-hour runtime limit.","deadline":"rolling","prizes":"recognition only","requirements":["Python","Docker","Sysbox","Kaggle API"],"compute":"datacenter-gpu"},"links":[{"label":"Paper","url":"https://arxiv.org/abs/2410.07095"},{"label":"GitHub","url":"https://github.com/openai/mle-bench"}],"organizer":{"name":"OpenAI","url":"https://openai.com/","type":"company"},"quickstart":"pip install -e .\nmlebench prepare --lite\n# Run your agent to generate a submission CSV, then grade it:\nmlebench grade-sample <PATH_TO_SUBMISSION> spaceship-titanic","problems":[{"metricUnit":"%","id":"low-complexity","name":"Low Complexity (Lite)","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","date":"2026-02-23","value":80.3,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-3-Pro-Preview","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"AIBuildAI","date":"2026-03-06","value":77.27,"contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","tool":"AIBuildAI","evidence":"Leaderboard column 'LLM(s) used' specifies Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"CAIR MARS+","date":"2026-02-17","value":78.79,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS+","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"MLEvolve","date":"2026-02-14","value":80.3,"contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"MLEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":80.3,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent 2.0","date":"2025-12-27","value":75.76,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"ML-Master 2.0","date":"2025-12-16","value":75.76,"contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","tool":"ML-Master 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"CAIR MARS","date":"2026-01-25","value":74.24,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":74.24,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Leeroo","date":"2025-12-07","value":68.18,"contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","tool":"Leeroo","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Thesis","date":"2025-11-10","value":65.15,"contributorKind":"method","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","tool":"Thesis","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5-codex","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","date":"2025-11-25","value":68.18,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.5","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent","date":"2025-10-10","value":62.12,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Operand ensemble","date":"2025-10-06","value":63.64,"contributorKind":"method","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","tool":"Operand","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","date":"2025-11-03","value":66.67,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"InternAgent","date":"2025-09-12","value":62.12,"contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","tool":"InternAgent","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-09-26","value":68.18,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"Neo multi-agent","date":"2025-07-28","value":48.48,"contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"model":"NEO","evidence":"NEO has achieved a new State-Of-The-Art on the MLE-Bench benchmark.","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","date":"2025-05-15","value":55,"contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","tool":"AIRA-dojo","evidence":"Leaderboard column 'LLM(s) used' specifies o3","family":"openai","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-08-15","value":51.52,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o3 + GPT-4.1","family":"openai","confidence":"high","role":"author"}},{"contributor":"ML-Master","date":"2025-06-17","value":48.48,"contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","tool":"ML-Master","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-05-14","value":48.18,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":35.91,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":18.55,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":19.7,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"OpenHands","date":"2024-10-08","value":12.12,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"OpenHands","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":10.23,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"author"}},{"contributor":"MLAB","date":"2024-10-08","value":4.55,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"MLAB","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"Disarray","date":"2026-02-03","value":90.91,"contributorKind":"method","contributorUrl":"https://disarray.ai/","isCurrentBest":true,"agent":{"model":"Claude-Opus-4.5, Claude-Sonnet-4.5, GPT-5.2-Codex, Gemini-3-Pro-Preview","tool":"Disarray","evidence":"Leaderboard column 'LLM(s) used' specifies Ensemble of Claude, GPT, and Gemini models","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"LoongFlow","date":"2026-02-09","value":77.27,"contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"model":"Gemini-3-Flash-Preview","tool":"LoongFlow","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Flash-Preview","family":"google","confidence":"high","role":"author"}}],"contested":true},{"metricUnit":"%","id":"medium-complexity","name":"Medium Complexity","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","date":"2026-02-23","value":64.04,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-3-Pro-Preview","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"AIBuildAI","date":"2026-03-06","value":61.4,"contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","tool":"AIBuildAI","evidence":"Leaderboard column 'LLM(s) used' specifies Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"CAIR MARS+","date":"2026-02-17","value":60.53,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS+","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"MLEvolve","date":"2026-02-14","value":57.89,"contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"MLEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":58.77,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent 2.0","date":"2025-12-27","value":57.89,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"ML-Master 2.0","date":"2025-12-16","value":50.88,"contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","tool":"ML-Master 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"CAIR MARS","date":"2026-01-25","value":52.63,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":45.61,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Leeroo","date":"2025-12-07","value":44.74,"contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","tool":"Leeroo","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Thesis","date":"2025-11-10","value":45.61,"contributorKind":"method","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","tool":"Thesis","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5-codex","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","date":"2025-11-25","value":34.21,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.5","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent","date":"2025-10-10","value":36.84,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Operand ensemble","date":"2025-10-06","value":33.33,"contributorKind":"method","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","tool":"Operand","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","date":"2025-11-03","value":25.44,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"InternAgent","date":"2025-09-12","value":26.32,"contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","tool":"InternAgent","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-09-26","value":21.05,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"Neo multi-agent","date":"2025-07-28","value":29.82,"contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"model":"NEO","evidence":"NEO has achieved a new State-Of-The-Art on the MLE-Bench benchmark.","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","date":"2025-05-15","value":21.97,"contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","tool":"AIRA-dojo","evidence":"Leaderboard column 'LLM(s) used' specifies o3","family":"openai","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-08-15","value":19.3,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o3 + GPT-4.1","family":"openai","confidence":"high","role":"author"}},{"contributor":"ML-Master","date":"2025-06-17","value":20.18,"contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","tool":"ML-Master","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-05-14","value":8.95,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":8.45,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":3.06,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":2.63,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"OpenHands","date":"2024-10-08","value":1.75,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"OpenHands","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":0.66,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"author"}},{"contributor":"MLAB","date":"2024-10-08","value":0,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"MLAB","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"Disarray","date":"2026-02-03","value":72.81,"contributorKind":"method","contributorUrl":"https://disarray.ai/","isCurrentBest":true,"agent":{"model":"Claude-Opus-4.5, Claude-Sonnet-4.5, GPT-5.2-Codex, Gemini-3-Pro-Preview","tool":"Disarray","evidence":"Leaderboard column 'LLM(s) used' specifies Ensemble of Claude, GPT, and Gemini models","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"LoongFlow","date":"2026-02-09","value":63.15,"contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"model":"Gemini-3-Flash-Preview","tool":"LoongFlow","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Flash-Preview","family":"google","confidence":"high","role":"author"}}],"contested":true},{"metricUnit":"%","id":"high-complexity","name":"High Complexity","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","date":"2026-02-23","value":42.22,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-3-Pro-Preview","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"AIBuildAI","date":"2026-03-06","value":46.67,"contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","tool":"AIBuildAI","evidence":"Leaderboard column 'LLM(s) used' specifies Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"CAIR MARS+","date":"2026-02-17","value":44.44,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS+","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"MLEvolve","date":"2026-02-14","value":42.22,"contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"MLEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":40,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent 2.0","date":"2025-12-27","value":40,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"ML-Master 2.0","date":"2025-12-16","value":42.22,"contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","tool":"ML-Master 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"CAIR MARS","date":"2026-01-25","value":37.78,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":35.55,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Leeroo","date":"2025-12-07","value":40,"contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","tool":"Leeroo","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Thesis","date":"2025-11-10","value":31.11,"contributorKind":"method","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","tool":"Thesis","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5-codex","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","date":"2025-11-25","value":33.33,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.5","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent","date":"2025-10-10","value":33.33,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Operand ensemble","date":"2025-10-06","value":20,"contributorKind":"method","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","tool":"Operand","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","date":"2025-11-03","value":31.11,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"InternAgent","date":"2025-09-12","value":24.44,"contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","tool":"InternAgent","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-09-26","value":22.22,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"Neo multi-agent","date":"2025-07-28","value":24.44,"contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"model":"NEO","evidence":"NEO has achieved a new State-Of-The-Art on the MLE-Bench benchmark.","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","date":"2025-05-15","value":21.67,"contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","tool":"AIRA-dojo","evidence":"Leaderboard column 'LLM(s) used' specifies o3","family":"openai","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-08-15","value":26.67,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o3 + GPT-4.1","family":"openai","confidence":"high","role":"author"}},{"contributor":"ML-Master","date":"2025-06-17","value":24.44,"contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","tool":"ML-Master","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-05-14","value":18.67,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":11.67,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":8.15,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":2.22,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"OpenHands","date":"2024-10-08","value":2.22,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"OpenHands","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":0,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"author"}},{"contributor":"MLAB","date":"2024-10-08","value":0,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"MLAB","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"Disarray","date":"2026-02-03","value":71.11,"contributorKind":"method","contributorUrl":"https://disarray.ai/","isCurrentBest":true,"agent":{"model":"Claude-Opus-4.5, Claude-Sonnet-4.5, GPT-5.2-Codex, Gemini-3-Pro-Preview","tool":"Disarray","evidence":"Leaderboard column 'LLM(s) used' specifies Ensemble of Claude, GPT, and Gemini models","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"LoongFlow","date":"2026-02-09","value":40,"contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"model":"Gemini-3-Flash-Preview","tool":"LoongFlow","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Flash-Preview","family":"google","confidence":"high","role":"author"}}],"contested":true},{"metricUnit":"%","id":"all-competitions","name":"All Competitions","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","date":"2026-02-23","value":64.44,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-3-Pro-Preview","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"AIBuildAI","date":"2026-03-06","value":63.11,"contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","tool":"AIBuildAI","evidence":"Leaderboard column 'LLM(s) used' specifies Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"CAIR MARS+","date":"2026-02-17","value":62.67,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS+","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"MLEvolve","date":"2026-02-14","value":61.33,"contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"MLEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":61.33,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent 2.0","date":"2025-12-27","value":59.56,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"ML-Master 2.0","date":"2025-12-16","value":56.44,"contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","tool":"ML-Master 2.0","evidence":"Leaderboard column 'LLM(s) used' specifies Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"CAIR MARS","date":"2026-01-25","value":56,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","tool":"MARS","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"PiEvolve","date":"2026-01-05","value":52,"contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","tool":"PiEvolve","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Leeroo","date":"2025-12-07","value":50.67,"contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","tool":"Leeroo","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"author"}},{"contributor":"Thesis","date":"2025-11-10","value":48.44,"contributorKind":"method","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","tool":"Thesis","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5-codex","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","date":"2025-11-25","value":44,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.5","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Famou-Agent","date":"2025-10-10","value":43.56,"contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","tool":"Famou-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"Operand ensemble","date":"2025-10-06","value":39.56,"contributorKind":"method","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","tool":"Operand","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","date":"2025-11-03","value":38.67,"contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","tool":"MLE-STAR-Pro-1.0","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-2.5-Pro","family":"google","confidence":"high","role":"author"}},{"contributor":"InternAgent","date":"2025-09-12","value":36.44,"contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","tool":"InternAgent","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-09-26","value":35.11,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-5","family":"openai","confidence":"high","role":"author"}},{"contributor":"Neo multi-agent","date":"2025-07-28","value":34.22,"contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"model":"NEO","evidence":"NEO has achieved a new State-Of-The-Art on the MLE-Bench benchmark.","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","date":"2025-05-15","value":31.6,"contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","tool":"AIRA-dojo","evidence":"Leaderboard column 'LLM(s) used' specifies o3","family":"openai","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-08-15","value":30.22,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o3 + GPT-4.1","family":"openai","confidence":"high","role":"author"}},{"contributor":"ML-Master","date":"2025-06-17","value":29.33,"contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","tool":"ML-Master","evidence":"Leaderboard column 'LLM(s) used' specifies deepseek-r1","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"R&D-Agent","date":"2025-05-14","value":22.4,"contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","tool":"R&D-Agent","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":17.12,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies o1-preview","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":8.63,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":7.56,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"OpenHands","date":"2024-10-08","value":4.89,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"OpenHands","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"AIDE","date":"2024-10-08","value":3.33,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","tool":"AIDE","evidence":"Leaderboard column 'LLM(s) used' specifies llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"author"}},{"contributor":"MLAB","date":"2024-10-08","value":1.6,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","tool":"MLAB","evidence":"Leaderboard column 'LLM(s) used' specifies gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"author"}},{"contributor":"Disarray","date":"2026-02-03","value":77.78,"contributorKind":"method","contributorUrl":"https://disarray.ai/","isCurrentBest":true,"agent":{"model":"Claude-Opus-4.5, Claude-Sonnet-4.5, GPT-5.2-Codex, Gemini-3-Pro-Preview","tool":"Disarray","evidence":"Leaderboard column 'LLM(s) used' specifies Ensemble of Claude, GPT, and Gemini models","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"LoongFlow","date":"2026-02-09","value":62.66,"contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"model":"Gemini-3-Flash-Preview","tool":"LoongFlow","evidence":"Leaderboard column 'LLM(s) used' specifies Gemini-3-Flash-Preview","family":"google","confidence":"high","role":"author"}}],"contested":true}],"agentStats":[{"family":"other-ai","records":4,"currentBests":4,"models":["Claude-Opus-4.5, Claude-Sonnet-4.5, GPT-5.2-Codex, Gemini-3-Pro-Preview"],"contributors":["Disarray"]},{"family":"google","records":48,"currentBests":0,"models":["Gemini-3-Pro-Preview","Gemini-2.5-Pro","Gemini-3-Flash-Preview"],"contributors":["Famou-Agent 2.0","CAIR MARS+","MLEvolve","PiEvolve","CAIR MARS","Leeroo","CAIR MLE-STAR-Pro-1.5","Famou-Agent","CAIR MLE-STAR-Pro-1.0","LoongFlow"]},{"family":"openai","records":40,"currentBests":0,"models":["gpt-5-codex","gpt-5","o3","o3 + GPT-4.1","o1-preview","gpt-4o-2024-08-06"],"contributors":["Thesis","Operand ensemble","R&D-Agent","AIRA-dojo","AIDE","OpenHands","MLAB"]},{"family":"deepseek","records":12,"currentBests":0,"models":["Deepseek-V3.2-Speciale","deepseek-r1"],"contributors":["ML-Master 2.0","InternAgent","ML-Master"]},{"family":"anthropic","records":8,"currentBests":0,"models":["Claude-Opus-4.6","claude-3-5-sonnet-20240620"],"contributors":["AIBuildAI","AIDE"]},{"family":"meta","records":4,"currentBests":0,"models":["llama-3.1-405b-instruct"],"contributors":["AIDE"]}],"repo":{"owner":"openai","name":"mle-bench","branch":"main"},"participants":[],"stats":{"totalRecords":120,"uniqueParticipants":0,"firstSubmission":"2024-10-08","lastSubmission":"2026-03-06","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.146Z"},{"id":"reference-kernels","name":"GPU Mode Reference Kernels","tagline":"Optimize high-performance GPU kernels for deep learning and linear algebra.","description":"A collection of reference kernels and optimization challenges hosted by GPU Mode. Developers optimize CUDA, Triton, and PyTorch kernels for speed and correctness across various hardware backends like NVIDIA and AMD, with performance profiled using Nsight Compute and PyTorch profilers.","url":"https://github.com/gpu-mode/reference-kernels","host":"github","status":"active","domain":"hardware-efficiency","tags":["cuda","triton","gpu-optimization","deep-learning","linear-algebra","kernel-tuning"],"images":[],"participation":{"howToSubmit":"Submit kernels via KernelBot on the GPU Mode Discord or use popcorn-cli to evaluate locally and submit.","submissionFormat":"Python/Triton/CUDA kernel files (e.g., submission.py) matching the problem's task.yml schema.","computeDetails":"Evaluated on NVIDIA (e.g., B200, A100) and AMD GPUs depending on the competition track.","verification":"Automated correctness checks against PyTorch reference implementations and performance profiling via Nsight Compute (NCU) or PyTorch profiler.","requirements":["Python","PyTorch","Triton","CUDA","Nsight Compute"],"compute":"datacenter-gpu"},"links":[{"label":"Website","url":"https://www.gpumode.com/"},{"label":"Discord","url":"https://discord.gg/gpumode"},{"label":"Popcorn CLI","url":"https://github.com/gpu-mode/popcorn-cli"}],"organizer":{"name":"GPU Mode","url":"https://www.gpumode.com/","type":"community"},"problems":[{"description":"Optimize the 1D causal convolution kernel, commonly used in modern sequence models like Mamba.","metricUnit":"ms","id":"causal-conv1d","name":"Causal Conv1d","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize the forward pass chunk output kernel for Gated Deltanet.","metricUnit":"ms","id":"gated-deltanet-chunk-fwd-o","name":"Gated Deltanet chunk_fwd_o","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize QR decomposition with a dedicated Nsight Compute-compatible profile path.","metricUnit":"ms","id":"qr-v2","name":"QR Decomposition v2","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize Cholesky decomposition with application validation over natural-gradient logistic-regression loops.","metricUnit":"ms","id":"cholesky","name":"Cholesky Validation","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize symmetric eigenvalue decomposition (eigh) validation kernels.","metricUnit":"ms","id":"eigh","name":"Eigh Validation","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"gpu-mode","name":"reference-kernels","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.146Z"},{"id":"llm-leaderboard","name":"LLM-Leaderboard","tagline":"A joint community effort to create one central leaderboard for LLMs.","description":"A community-curated leaderboard tracking the performance of open and closed large language models across multiple benchmarks. It aggregates results from Chatbot Arena, HellaSwag, HumanEval, LAMBADA, MMLU, TriviaQA, and WinoGrande. Anyone can contribute by opening a pull request to add new models or fill in missing evaluation scores.","url":"https://github.com/LudwigStumpp/llm-leaderboard","host":"github","status":"ended","domain":"llm-eval","tags":["llm","leaderboard","nlp","evaluation","benchmark"],"images":[],"participation":{"howToSubmit":"Open a PR to main adding a new model row or filling missing entries in the README.md table.","submissionFormat":"Markdown table row update with links to sources for each reported value.","computeDetails":"No compute is run by the repository; contributors submit pre-computed scores from official papers or published results.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"human review","requirements":["Markdown"],"compute":"none"},"links":[{"label":"Interactive Dashboard","url":"https://llm-leaderboard.streamlit.app/"},{"label":"HuggingFace Space","url":"https://huggingface.co/spaces/ludwigstumpp/llm-leaderboard"}],"organizer":{"name":"Ludwig Stumpp","url":"https://github.com/LudwigStumpp","type":"individual"},"problems":[{"metricUnit":"Elo","id":"chatbot-arena-elo","name":"Chatbot Arena Elo","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"alpaca-13b","value":1008,"contributorKind":"model","contributorUrl":"https://crfm.stanford.edu/2023/03/13/alpaca.html","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"alpaca-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chatglm-6b","value":985,"contributorKind":"model","contributorUrl":"https://chatglm.cn/blog","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"chatglm-6b","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"dolly-v2-12b","value":944,"contributorKind":"model","contributorUrl":"https://huggingface.co/databricks/dolly-v2-12b","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"dolly-v2-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"fastchat-t5-3b","value":951,"contributorKind":"model","contributorUrl":"https://huggingface.co/lmsys/fastchat-t5-3b-v1.0","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"fastchat-t5-3b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"koala-13b","value":1082,"contributorKind":"model","contributorUrl":"https://bair.berkeley.edu/blog/2023/04/03/koala/","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"koala-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":932,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"llama-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"oasst-pythia-12b","value":1065,"contributorKind":"model","contributorUrl":"https://huggingface.co/OpenAssistant/pythia-12b-pre-v8-12.5k-steps","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"oasst-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-tuned-alpha-7b","value":858,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"model":"stablelm-tuned-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"vicuna-13b","value":1169,"contributorKind":"model","contributorUrl":"https://huggingface.co/lmsys/vicuna-13b-delta-v0","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"isCurrentBest":true,"agent":{"model":"vicuna-13b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"hellaswag-few-shot","name":"HellaSwag (few-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"bloom-176b","value":0.744,"contributorKind":"model","contributorUrl":"https://huggingface.co/bigscience/bloom","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"bloom-176b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"falcon-7b","value":0.781,"contributorKind":"model","contributorUrl":"https://huggingface.co/tiiuae/falcon-40b","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"model":"falcon-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"falcon-40b","value":0.853,"contributorKind":"model","contributorUrl":"https://huggingface.co/tiiuae/falcon-40b","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"model":"falcon-40b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-7b / curie","value":0.682,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2005.14165","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"gpt-3-7b / curie","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.793,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2005.14165","submissionUrls":["https://arxiv.org/abs/2005.14165"],"agent":{"model":"gpt-3-175b / davinci","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.822,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-4","value":0.953,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"isCurrentBest":true,"agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.718,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-neox-20b","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"gpt-neox-20b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.663,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-j-6b","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"gpt-j-6b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-2-70b","value":0.873,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2307.09288","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"model":"llama-2-70b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-66b","value":0.745,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-66b","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"opt-66b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-175b","value":0.791,"contributorKind":"model","contributorUrl":"https://ai.facebook.com/blog/democratizing-access-to-large-scale-language-models-with-opt-175b/","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"opt-175b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.838,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"id":"hellaswag-zero-shot","name":"HellaSwag (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"alpaca-7b","value":0.739,"contributorKind":"model","contributorUrl":"https://crfm.stanford.edu/2023/03/13/alpaca.html","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"alpaca-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-7b","value":0.636,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-6.7B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.635,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-13B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chinchilla-70b","value":0.808,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2203.15556v1","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"model":"chinchilla-70b","family":"google","confidence":"high","role":"subject"}},{"contributor":"dolly-v2-12b","value":0.71,"contributorKind":"model","contributorUrl":"https://huggingface.co/databricks/dolly-v2-12b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"dolly-v2-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.667,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-6.9b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.704,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-12b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.789,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2005.14165","submissionUrls":["https://arxiv.org/abs/2005.14165"],"agent":{"model":"gpt-3-175b / davinci","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.834,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt4all-13b-snoozy","value":0.75,"contributorKind":"model","contributorUrl":"https://huggingface.co/nomic-ai/gpt4all-13b-snoozy","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"gpt4all-13b-snoozy","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.719,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-neox-20b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-neox-20b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.683,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-j-6b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-j-6b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"koala-13b","value":0.726,"contributorKind":"model","contributorUrl":"https://bair.berkeley.edu/blog/2023/04/03/koala/","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"koala-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.738,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"llama-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":0.792,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971"],"agent":{"model":"llama-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-33b","value":0.828,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971"],"agent":{"model":"llama-33b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-65b","value":0.842,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971"],"isCurrentBest":true,"agent":{"model":"llama-65b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.761,"contributorKind":"model","contributorUrl":"https://huggingface.co/mosaicml/mpt-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"mpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"oasst-pythia-12b","value":0.681,"contributorKind":"model","contributorUrl":"https://huggingface.co/OpenAssistant/pythia-12b-pre-v8-12.5k-steps","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"oasst-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.677,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-6.7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.692,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-13b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.834,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.412,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-base-alpha-7b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"stablelm-base-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-tuned-alpha-7b","value":0.536,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"stablelm-tuned-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"hellaswag-one-shot","name":"HellaSwag (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-540b","value":0.836,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.82,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-s","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.84,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-m","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-l","value":0.868,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"model":"palm-2-l","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"id":"humaneval-python-pass1","name":"HumanEval-Python (pass@1)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"bloom-176b","value":0.155,"contributorKind":"model","contributorUrl":"https://huggingface.co/bigscience/bloom","submissionUrls":["https://huggingface.co/bigscience/bloom#results"],"agent":{"model":"bloom-176b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"codex-12b / code-cushman-001","value":0.317,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2107.03374","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=targeted_evaluations"],"agent":{"model":"codex-12b / code-cushman-001","family":"openai","confidence":"high","role":"subject"}},{"contributor":"codegen-16B-mono","value":0.293,"contributorKind":"model","contributorUrl":"https://huggingface.co/Salesforce/codegen-16B-mono","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"codegen-16B-mono","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"codegen-16B-multi","value":0.183,"contributorKind":"model","contributorUrl":"https://huggingface.co/Salesforce/codegen-16B-multi","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"codegen-16B-multi","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"codegx-13b","value":0.229,"contributorKind":"model","contributorUrl":"http://keg.cs.tsinghua.edu.cn/codegeex/","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"codegx-13b","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.481,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / code-davinci-002","value":0.463,"contributorKind":"model","contributorUrl":"https://platform.openai.com/docs/model-index-for-researchers","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=targeted_evaluations"],"agent":{"model":"gpt-3.5-175b / code-davinci-002","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-4","value":0.67,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"isCurrentBest":true,"agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.105,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"llama-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":0.158,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"llama-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-33b","value":0.217,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"llama-33b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-65b","value":0.237,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"llama-65b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.262,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-coder-540b","value":0.359,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-coder-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s*","value":0.376,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-s*","family":"google","confidence":"high","role":"subject"}},{"contributor":"replit-code-v1-3b","value":0.219,"contributorKind":"model","contributorUrl":"https://huggingface.co/replit/replit-code-v1-3b","submissionUrls":["https://twitter.com/amasad/status/1651019556423598081/photo/2"],"agent":{"model":"replit-code-v1-3b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"starcoder-base-16b","value":0.304,"contributorKind":"model","contributorUrl":"https://huggingface.co/bigcode/starcoderbase","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"starcoder-base-16b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"starcoder-16b","value":0.336,"contributorKind":"model","contributorUrl":"https://huggingface.co/bigcode/starcoder","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"model":"starcoder-16b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"lambada-zero-shot","name":"LAMBADA (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"cerebras-gpt-7b","value":0.636,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-6.7B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.635,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-13B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chinchilla-70b","value":0.774,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2203.15556v1","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"model":"chinchilla-70b","family":"google","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.667,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-6.9b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.704,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-12b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.762,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.719,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-neox-20b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-neox-20b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.683,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-j-6b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-j-6b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.738,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"llama-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.702,"contributorKind":"model","contributorUrl":"https://huggingface.co/mosaicml/mpt-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"mpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.677,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-6.7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.692,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-13b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.779,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"isCurrentBest":true,"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.533,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-base-alpha-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"stablelm-base-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"lambada-one-shot","name":"LAMBADA (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-540b","value":0.818,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.807,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-s","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.837,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-m","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-l","value":0.869,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"model":"palm-2-l","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"id":"mmlu-zero-shot","name":"MMLU (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"bloom-176b","value":0.299,"contributorKind":"model","contributorUrl":"https://huggingface.co/bigscience/bloom","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"bloom-176b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-7b","value":0.259,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-6.7B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.258,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-13B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.265,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-6.9b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.253,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-12b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gal-120b","value":0.526,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2211.09085v1","submissionUrls":["https://paperswithcode.com/paper/galactica-a-large-language-model-for-science-1"],"isCurrentBest":true,"agent":{"model":"gal-120b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.269,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-neox-20b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-neox-20b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.261,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-j-6b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-j-6b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.302,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"llama-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.296,"contributorKind":"model","contributorUrl":"https://huggingface.co/mosaicml/mpt-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"mpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.251,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-6.7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.257,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-13b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.251,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-base-alpha-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"stablelm-base-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"mmlu-few-shot","name":"MMLU (few-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"falcon-7b","value":0.35,"contributorKind":"model","contributorUrl":"https://huggingface.co/tiiuae/falcon-40b","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"model":"falcon-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"falcon-40b","value":0.527,"contributorKind":"model","contributorUrl":"https://huggingface.co/tiiuae/falcon-40b","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"model":"falcon-40b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-7b / curie","value":0.243,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2005.14165","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"gpt-3-7b / curie","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.439,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2005.14165","submissionUrls":["https://arxiv.org/abs/2005.14165"],"agent":{"model":"gpt-3-175b / davinci","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.569,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-4","value":0.864,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"isCurrentBest":true,"agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.276,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-neox-20b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-neox-20b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.249,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-j-6b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-j-6b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-65b","value":0.634,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"model":"llama-65b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-2-70b","value":0.698,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2307.09288","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"model":"llama-2-70b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-66b","value":0.276,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-66b","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"opt-66b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-175b","value":0.318,"contributorKind":"model","contributorUrl":"https://ai.facebook.com/blog/democratizing-access-to-large-scale-language-models-with-opt-175b/","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"model":"opt-175b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.693,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"id":"triviaqa-zero-shot","name":"TriviaQA (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"cerebras-gpt-7b","value":0.141,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-6.7B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.146,"contributorKind":"model","contributorUrl":"https://huggingface.co/cerebras/Cerebras-GPT-13B","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"cerebras-gpt-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.198,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-6.9b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.233,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-12b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"eleuther-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.347,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-neox-20b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-neox-20b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.234,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/gpt-j-6b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"gpt-j-6b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.443,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"isCurrentBest":true,"agent":{"model":"llama-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.343,"contributorKind":"model","contributorUrl":"https://huggingface.co/mosaicml/mpt-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"mpt-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.227,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-6.7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.282,"contributorKind":"model","contributorUrl":"https://huggingface.co/facebook/opt-13b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"opt-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.049,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-base-alpha-7b","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"model":"stablelm-base-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"triviaqa-one-shot","name":"TriviaQA (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-540b","value":0.814,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.752,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-s","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.817,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-m","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-l","value":0.861,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"model":"palm-2-l","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"id":"winogrande-zero-shot","name":"WinoGrande (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"alpaca-7b","value":0.661,"contributorKind":"model","contributorUrl":"https://crfm.stanford.edu/2023/03/13/alpaca.html","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"alpaca-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chinchilla-70b","value":0.749,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2203.15556v1","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"model":"chinchilla-70b","family":"google","confidence":"high","role":"subject"}},{"contributor":"dolly-v2-12b","value":0.622,"contributorKind":"model","contributorUrl":"https://huggingface.co/databricks/dolly-v2-12b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"dolly-v2-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.661,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-6.9b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"eleuther-pythia-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.638,"contributorKind":"model","contributorUrl":"https://huggingface.co/EleutherAI/pythia-12b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"eleuther-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.702,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2005.14165","submissionUrls":["https://arxiv.org/abs/2005.14165v4"],"agent":{"model":"gpt-3-175b / davinci","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.758,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt4all-13b-snoozy","value":0.713,"contributorKind":"model","contributorUrl":"https://huggingface.co/nomic-ai/gpt4all-13b-snoozy","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"gpt4all-13b-snoozy","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"koala-13b","value":0.688,"contributorKind":"model","contributorUrl":"https://bair.berkeley.edu/blog/2023/04/03/koala/","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"koala-13b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.701,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"model":"llama-7b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":0.73,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"model":"llama-13b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-33b","value":0.76,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"model":"llama-33b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-65b","value":0.77,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2302.13971","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"model":"llama-65b","family":"meta","confidence":"high","role":"subject"}},{"contributor":"oasst-pythia-12b","value":0.65,"contributorKind":"model","contributorUrl":"https://huggingface.co/OpenAssistant/pythia-12b-pre-v8-12.5k-steps","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"oasst-pythia-12b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"palm-62b","value":0.77,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311"],"agent":{"model":"palm-62b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.811,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311"],"isCurrentBest":true,"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.501,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-base-alpha-7b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"stablelm-base-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-tuned-alpha-7b","value":0.548,"contributorKind":"model","contributorUrl":"https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"model":"stablelm-tuned-alpha-7b","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"id":"winogrande-one-shot","name":"WinoGrande (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-540b","value":0.837,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.779,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-s","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.792,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-m","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-l","value":0.83,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"model":"palm-2-l","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"id":"winogrande-few-shot","name":"WinoGrande (few-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.816,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"model":"gpt-3.5-175b / text-davinci-003","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-4","value":0.875,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2303.08774v3","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"model":"gpt-4","family":"openai","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.851,"contributorKind":"model","contributorUrl":"https://arxiv.org/abs/2204.02311v5","submissionUrls":["https://arxiv.org/abs/2204.02311"],"agent":{"model":"palm-540b","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-l-instruct","value":0.909,"contributorKind":"model","contributorUrl":"https://ai.google/static/documents/palm2techreport.pdf","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"model":"palm-2-l-instruct","family":"google","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"LudwigStumpp","name":"llm-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":153,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.148Z"},{"id":"medmcqa","name":"MedMCQA","tagline":"A large-scale multiple-choice question answering dataset for the medical domain.","description":"MedMCQA is a large-scale, Multiple-Choice Question Answering dataset designed to address real-world medical entrance exam questions (AIIMS & NEET PG). It covers 2.4k healthcare topics and 21 medical subjects, testing reasoning abilities across a wide range of medical subjects. Submissions are evaluated on the private test set via a Google Form submission.","url":"https://github.com/medmcqa/medmcqa","host":"github","status":"active","domain":"llm-eval","tags":["medical-qa","mcqa","nlp","benchmark","llm-eval"],"images":[],"participation":{"howToSubmit":"Evaluate your model on the test set and submit predictions in CSV format via the Google Form.","submissionFormat":"CSV file with columns 'id' and 'Prediction (correct option)' (1-indexed).","deadline":"rolling","cost":"free","requirements":["Python"],"compute":"unknown"},"links":[{"label":"Website","url":"https://medmcqa.github.io/"},{"label":"Paper","url":"https://arxiv.org/abs/2203.14371"},{"label":"Leaderboard","url":"https://paperswithcode.com/dataset/medmcqa"},{"label":"Submission Form","url":"https://forms.gle/xLJHNbuvaRa2FXbD8"}],"organizer":{"name":"MedMCQA Team","type":"community"},"quickstart":"python3 train.py --model bert-base-uncased --dataset_folder_name \"/content/medmcqa_data/\"","problems":[{"description":"Accuracy on the private test set consisting of AIIMS PG exam MCQs.","metricUnit":"%","id":"medmcqa-test-accuracy","name":"MedMCQA Test Accuracy","metricName":"accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"medmcqa","name":"medmcqa","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.148Z"},{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","tagline":"Highscores and world records for the mobile game Grand Mountain Adventure.","description":"A community-driven leaderboard tracking highscores, world records, and player rankings for Grand Mountain Adventure. Players submit their best times and scores across over 200 in-game challenges via a Google Form. The platform features custom point ratings, world record histories, and player comparisons.","url":"https://github.com/hschindele/gmaapp","host":"github","status":"active","domain":"other","tags":["grand mountain adventure","leaderboard","speedrun","highscores","mobile-gaming"],"images":[],"participation":{"howToSubmit":"Submit your scores using the Google Form.","submissionFormat":"Google Form submission with a link to proof (e.g., Google Drive screenshot/video).","cost":"free","verification":"human review","requirements":["Grand Mountain Adventure game (iOS or Android)"],"compute":"none"},"links":[{"label":"Leaderboard Website","url":"https://gmaleaderboard.gart.sh/"},{"label":"Score Submission Form","url":"https://docs.google.com/forms/d/1n3DL3h3NYxhWCBG0NSrtM48uIE3OGvMfL2MVKdBU_kA/edit#responses"},{"label":"Community Discord","url":"https://discord.com/invite/SA4F5WN"},{"label":"Toppluva Website","url":"https://www.toppluva.com/press/index.php"}],"organizer":{"name":"hschindele","type":"individual"},"problems":[{"group":"Agpat Island","description":"Time trial race on Agpat Island.","metricUnit":"s","id":"agpat-journey","name":"Agpat Journey","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"hschindele","date":"2022-03-10","value":22.12,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1AqfqRGTJlN7B-etugqSr9kpfd1wOfj20"],"isCurrentBest":false},{"contributor":"Jamiemac","date":"2022-03-11","value":22.71,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1yscpPqeKfBk9T1O3KQRoPjMQMq3RAE91"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":22.82,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1S02qi45OK_A68ac8YeloQQWV4kBXmLZ7"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":22.15,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1WPV3-iUZwwfcueGHImTtKTk-UFrWGKh3"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-27","value":21.31,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1ZFYiKLgZIIh3SdwxSHOluMmq4LyqJemi"],"isCurrentBest":true}],"contested":true},{"group":"Agpat Island","description":"Time trial race on Agpat Island.","metricUnit":"s","id":"aluu-race-track","name":"Aluu Race Track","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":20.12,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1o4nqAfAhAGNamesJ_yvwj3VwE7Jry4Io"],"isCurrentBest":false},{"contributor":"Jamiemac","date":"2022-03-11","value":20.1,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1pSL7sAe6GYtMVrEhGMz4PLTeSWO9YCBo"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-03-12","value":20.08,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1aqDDDACbEv4AVpFL1w7elcekZnGa5nnk"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":20.1,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=15AhCX1iCL1_KXy6SI8DqWFNiTOSU7IdB"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":20.2,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1fwL6O4q6Yg0Zzjg3zHhCft0EIayXdj6-"],"isCurrentBest":false},{"contributor":"Wrog","date":"2022-04-13","value":20.22,"contributorKind":"person","contributorUrl":"https://github.com/Wrog","submissionUrls":["https://drive.google.com/open?id=1eB5JW0ehJnWYNAnk_KIbxO8cohthQ167"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-04-14","value":20.06,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1dhWpOJSCIoNnP2CxRHHueBZ7y8qlh2T2"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-04-17","value":20.04,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1kPfwBoK8rk6zBZpDyKTPaZ3uNurqgdqS"],"isCurrentBest":true},{"contributor":"Wrog","date":"2022-06-27","value":20.14,"contributorKind":"person","contributorUrl":"https://github.com/Wrog","submissionUrls":["https://drive.google.com/open?id=1JZBj_pXpe-fxfKQno3yz7Z9Gw3BuQZxO"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"Time trial race on Agpat Island.","metricUnit":"s","id":"ilineq-track","name":"Ilineq Track","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":15.37,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1kG3lQaaCHcWBSjDJz-FLP2cVHkxALuc0"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-03-12","value":15.35,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=12sQImIbjcekf-0JJGL1tSLebuK8WUwEm"],"isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":15.49,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1qCwrvRIkHeWy4jxn8VfBYZ_mJ9A35SgP"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":15.41,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1JCLk3kALUBViLuDcdY8CbmFOl9shDaR2"],"isCurrentBest":false},{"contributor":"Wrog","date":"2022-06-27","value":15.47,"contributorKind":"person","contributorUrl":"https://github.com/Wrog","submissionUrls":["https://drive.google.com/open?id=1SM6vKRycr9JSOmKxlYszNqX92typk4NU"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"Time trial race on Agpat Island.","metricUnit":"s","id":"nanoq-danger","name":"Nanoq danger","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":21.99,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1BHxI2qZAeWwOPgszMYYQzKBTqupuj42u"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-03-12","value":21.29,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1l0bU_rDT5_ye2NmM17Bg3ue_1V5gmbwD"],"isCurrentBest":false},{"contributor":"Jonathan","date":"2022-03-15","value":21.1,"contributorKind":"person","contributorUrl":"https://github.com/Jonathan","submissionUrls":["https://drive.google.com/open?id=1sJhVYo5C7MEEgUHSYH51ai8wj0XvT9p2"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":21.77,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1pqt1May40BZ3GBd-W3KI9WzyO8UWMhMS"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":21.72,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1b4RswuSxoursQrZ3oBaUF5-9dlaDL2Jq"],"isCurrentBest":false},{"contributor":"Jamiemac","date":"2022-03-27","value":21.87,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1SlqzC3wlQUEiffxZhaZ4HbQr08QzmpIy"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-27","value":20.97,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1nWiB5Zh4XWH0un4LABEQ3kSr37-gwA-b"],"isCurrentBest":true}],"contested":true},{"group":"Agpat Island","description":"Time trial race on Agpat Island.","metricUnit":"s","id":"the-pilot-hike","name":"The Pilot Hike","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":16.75,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1Cvj8BhGJTCtLAOzEtcjDRWC9gESVoaWX"],"isCurrentBest":true},{"contributor":"hschindele","date":"2022-03-12","value":16.89,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1HnGpRbT1JvGKGYeHlr8_EX-KQMHUiTfs"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":17.07,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1Lo3bItH4iuZDJM5FeETYMEGzV_5Mp809"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":17.06,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1ModS6-U07yAwk5WIBbGrPvr38nMm_wzG"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-27","value":16.95,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1dROkiY_rPZEE_Z8CvqZfOjT7xHNUhHWV"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"Time trial race on Agpat Island.","metricUnit":"s","id":"ukioq-track","name":"Ukioq Track","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":15.4,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1ByAGoFJvsmmx_N8Gswb5qcCLOoiiuQsn"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-03-12","value":15.36,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1NF_00bQpMd7C5Jg53wsmRzmombFtqsZz"],"isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":15.73,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1nrHaE4TL5KKDjmzhJRDdIAXxuTFcDO6M"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":15.47,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1ui2pp-uRd76cQ73YDZMBX44Tl-nHqlcX"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"Time trial trail on Agpat Island.","metricUnit":"s","id":"gunnbjorns-line","name":"Gunnbjorns Line","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":22.58,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=16AlLcYybq-7JE1XcXA-G7OnVWkqYb5dO"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-03-12","value":22.4,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=15pKkwfp-warVPLgTBAaGKXAMmWv8H6Pk"],"isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":22.55,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1L-Oz8lGAQ3p_vZS6sl5-mPdUZmj5xFi2"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":22.49,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1I6JCnGlVWQtFi9xrlxbhlzrgdGbBegRN"],"isCurrentBest":false},{"contributor":"Jamiemac","date":"2022-04-07","value":22.55,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1HXxKXbUCCx5ochkL3GXO2WkMOB0At7EE"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"Time trial TTB on Agpat Island.","metricUnit":"s","id":"across-agpat","name":"Across Agpat","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":19.7,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=12Xit_jZD_xIRW6q7uGPBIAcpR86elHWj"],"isCurrentBest":false},{"contributor":"hschindele","date":"2022-03-12","value":18.9,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1b-MrqDjx8cscAavqZS83Ky9edSVUTCWJ"],"isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":19.81,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1MLXGBgqBNhKXnL1Oa7omkuf2obGDXRuc"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":19.15,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1d70nS9EcCIwbonxUzA2tQi0SIvqyCfwr"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"High score airtime challenge on Agpat Island.","id":"the-brand","name":"The Brand","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":10.26,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1FnJJgFmBS9RZ6xHjowdhby0cmdEZBc6y"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":9.08,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1vZZm_rvGflCJsN1Tt1joE9yqEmJe0JU9"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":10.26,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1KnQaRa9VgvekC192SzH7gas7uecE_5l-"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-26","value":10.32,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1lygW8vjVJZo38yUfVwRQSnIcMNLaPHwv"],"isCurrentBest":false},{"contributor":"Jonathan","date":"2022-04-07","value":11.26,"contributorKind":"person","contributorUrl":"https://github.com/Jonathan","submissionUrls":["https://drive.google.com/open?id=1FX070iHRhsA2FrrhA11yQ3f_GN4BFoSL"],"isCurrentBest":true},{"contributor":"hschindele","date":"2022-04-17","value":10.46,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","submissionUrls":["https://drive.google.com/open?id=1aY9MHQGb7fwTdsvgKKuIi0CcWxADSKo-"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"High score long jump challenge on Agpat Island.","id":"the-seagull","name":"The Seagull","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":80.7,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1zlSZOwMZ_wi92vh2qBRGZYlb-UQudUnJ"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":87.8,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1LstdaJNGhO4MU0JFWol5dC2X6J_LqUhE"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":88.1,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1pqmqxWbfYzKfekimR0PqvrUAb0qw56Up"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-26","value":93.9,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1wrusThAjTMju6RQKKCNXqwfL4lXzTRm4"],"isCurrentBest":true},{"contributor":"Jamiemac","date":"2022-04-07","value":84.8,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1emUUL2t0mgcFHFfRRrx2aXjmcX5WSOqo"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-04-14","value":90.6,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1H2vFNSoeCrjpMwQsLzCXUF5fP4hTNTTq"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"High score single trick challenge on Agpat Island.","id":"greenland-spin","name":"Greenland spin","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":18396,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1GiR3XTDB6apnBcAm0D8kIMg2CuD4DVb0"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":21683,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=171a2ASxYEkWLhqMx1fL33484PPjNB-S3"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":20106,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1J5DOkyV6yZXjYAswyuZdKJI5Z44QGmye"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-04-21","value":23014,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1LS0VC7vXLOw1dsOZNqwUQJ9iIizcksUJ"],"isCurrentBest":true}],"contested":true},{"group":"Agpat Island","description":"High score gated trick challenge on Agpat Island.","id":"aappaluttoq-park-lap","name":"Aappaluttoq Park Lap","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":41380,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=143FVx2hGUeAK6_ewBYK3Xs1F_zEXty-O"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":38530,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1661UZuk0swgfykaULm1Q1xhh05f7iihY"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":35057,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1TfcfBHFX029faCOVOlyIHKRZArbdi_pF"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-26","value":41488,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1akgEsjRhOsJQ3XP5vBwuuToaKxRu4Q40"],"isCurrentBest":true},{"contributor":"Mountain_Greg","date":"2022-04-20","value":38167,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1l1g9kgUvaP84SKs0gvVP5vd4BAXKTYNx"],"isCurrentBest":false}],"contested":true},{"group":"Agpat Island","description":"High score gated trick challenge on Agpat Island.","id":"quick-turns","name":"Quick Turns","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":16839,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1rvEzx7xhEThjTb9Y5K73CAXIUvtPp_eO"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":12521,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1ZT8OWlbOSNe1Nw6mf0no8vwB1x-hjuEH"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":17760,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=113jSqD_in_T0XnUz6V6nDDP_Am7b9Kit"],"isCurrentBest":true}],"contested":true},{"group":"Agpat Island","description":"High score gated trick challenge on Agpat Island.","id":"tungujortoq-park-lap","name":"Tungujortoq Park Lap","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":28509,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","submissionUrls":["https://drive.google.com/open?id=1VHQ2xWvJUy_W4bDkMSIPsN2ejoV1rf9w"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-21","value":25148,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=1oPVjbLYFkQOmvpNk3WBeabBP6IP4BO3u"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-03-24","value":26887,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=1A_2Tik_jcwyr4LxtFCzKKIM9q9g59h7v"],"isCurrentBest":false},{"contributor":"Avel","date":"2022-03-26","value":33673,"contributorKind":"person","contributorUrl":"https://github.com/Avel","submissionUrls":["https://drive.google.com/open?id=16xucDW8-vtd20M2_0r4-fqdsoNwXUB71"],"isCurrentBest":false},{"contributor":"Mountain_Greg","date":"2022-04-21","value":28249,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","submissionUrls":["https://drive.google.com/open?id=11K7ruMErS1K7g5jKWzZ7A7An_zVUt2qQ"],"isCurrentBest":false},{"contributor":"GSM","date":"2022-08-08","value":34478,"contributorKind":"person","contributorUrl":"https://github.com/GSM","submissionUrls":["https://drive.google.com/open?id=1oychemuqC5lYe6LmAPI_nWVv05DJaNDe"],"isCurrentBest":true}],"contested":true}],"agentStats":[],"repo":{"owner":"hschindele","name":"gmaapp","branch":"main"},"participants":[{"name":"Avel","url":"https://github.com/Avel","submissions":21,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand","the-seagull","greenland-spin","aappaluttoq-park-lap","quick-turns","tungujortoq-park-lap"],"lastActive":"2022-03-27","bestRank":1},{"name":"Jamiemac","url":"https://github.com/Jamiemac","submissions":18,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand","the-seagull","greenland-spin","aappaluttoq-park-lap","quick-turns","tungujortoq-park-lap"],"lastActive":"2022-04-07","bestRank":1},{"name":"Mountain_Greg","url":"https://github.com/Mountain_Greg","submissions":18,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand","the-seagull","greenland-spin","aappaluttoq-park-lap","quick-turns","tungujortoq-park-lap"],"lastActive":"2022-04-21","bestRank":1},{"name":"hschindele","url":"https://github.com/hschindele","submissions":11,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand"],"lastActive":"2022-04-17","bestRank":1},{"name":"Jonathan","url":"https://github.com/Jonathan","submissions":2,"problems":["nanoq-danger","the-brand"],"lastActive":"2022-04-07","bestRank":1},{"name":"GSM","url":"https://github.com/GSM","submissions":1,"problems":["tungujortoq-park-lap"],"bestRank":1,"lastActive":"2022-08-08"},{"name":"Wrog","url":"https://github.com/Wrog","submissions":3,"problems":["aluu-race-track","ilineq-track"],"lastActive":"2022-06-27"}],"stats":{"totalRecords":74,"uniqueParticipants":7,"firstSubmission":"2022-03-10","lastSubmission":"2022-08-08","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.149Z"},{"id":"terminal-bench-leaderboard","name":"Terminal Bench 2.0 Leaderboard","tagline":"Evaluate agents on terminal-based coding and system tasks.","description":"Terminal Bench 2.0 is a benchmark that evaluates AI agents on terminal-based coding and system tasks. Agents are scored by task pass rate, which is the percentage of tasks completed successfully. Submissions are run automatically via GitHub Actions using Docker Compose.","url":"https://github.com/RDI-Foundation/terminal-bench-leaderboard","host":"github","status":"active","domain":"coding-agents","tags":["terminal","coding-agents","benchmark","agentbeats","docker"],"images":[],"participation":{"howToSubmit":"Fork the repository, fill in agent details in scenario.toml, and push to trigger the scenario runner. Results are submitted as a pull request.","submissionFormat":"scenario.toml configuration","computeDetails":"Parallel evaluation can be configured across multiple GitHub runners by editing num_shards in quick-submit.yml.","deadline":"rolling","cost":"free","verification":"CI","requirements":["Docker","Docker Compose","GitHub Actions","API keys"],"compute":"unknown"},"links":[{"label":"Terminal Bench Repository","url":"https://github.com/laude-ntt/terminal-bench"},{"label":"AgentBeats","url":"https://agentbeats.dev/"}],"organizer":{"name":"RDI Foundation","url":"https://github.com/RDI-Foundation","type":"foundation"},"quickstart":"To submit your agent, fork the repo and configure scenario.toml:\n\n[[participants]]\nagentbeats_id = \"YOUR_AGENT_ID\"\nAGENT_LLM = \"openai/gpt-4o\"","problems":[{"description":"Overall task pass rate across all terminal-based coding and system tasks.","metricUnit":"%","id":"terminal-bench-2.0","name":"Terminal Bench 2.0","metricName":"pass_rate","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"019db41d-2a0a-7240-a4df-ae1b55065c3d","description":"Submission ID: 019dd98c-29c2-7621-98a6-bd6622066879","date":"2026-04-29","value":0,"contributorKind":"unknown","isCurrentBest":true,"agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019db41d-2a0a-7240-a4df-ae1b55065c3d","description":"Submission ID: 019dd953-c6d9-7da0-8171-b4361369bc05","date":"2026-04-29","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019db41d-2a0a-7240-a4df-ae1b55065c3d","description":"Submission ID: 019dd928-6d99-7973-8d67-ff55b1f69e7f","date":"2026-04-29","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76ea-e6cd-7041-aef5-f7ee9d57ce58","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76da-437f-7c33-a037-569015b1272f","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76d0-5e6e-7f71-ac4a-39996fef99c8","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76c7-6d7e-7bc3-aa66-0e1d80fc0067","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"RDI-Foundation","name":"terminal-bench-leaderboard","branch":"main"},"participants":[{"name":"019db41d-2a0a-7240-a4df-ae1b55065c3d","submissions":3,"problems":["terminal-bench-2.0"],"bestRank":1,"lastActive":"2026-04-29"},{"name":"019d0393-4eac-7232-ba9b-865e8c539e35","submissions":4,"problems":["terminal-bench-2.0"],"lastActive":"2026-04-10"}],"stats":{"totalRecords":7,"uniqueParticipants":2,"firstSubmission":"2026-04-10","lastSubmission":"2026-04-29","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-13T12:55:04.149Z"}],"people":[{"name":"hschindele","url":"https://github.com/hschindele","records":11,"currentBests":5,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":11,"bests":5}],"agents":{},"lastActive":"2022-04-17"},{"name":"Avel","url":"https://github.com/Avel","records":21,"currentBests":4,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":21,"bests":4}],"agents":{},"lastActive":"2022-03-27"},{"name":"jurajselep","url":"https://github.com/jurajselep","records":12,"currentBests":3,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":12,"bests":3}],"agents":{},"lastActive":"2026-09-18"},{"name":"npow","url":"https://github.com/npow","records":9,"currentBests":3,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":9,"bests":3}],"agents":{},"lastActive":"2026-09-01"},{"name":"Mountain_Greg","url":"https://github.com/Mountain_Greg","records":18,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":18,"bests":2}],"agents":{},"lastActive":"2022-04-21"},{"name":"Jamiemac","url":"https://github.com/Jamiemac","records":18,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":18,"bests":1}],"agents":{},"lastActive":"2022-04-07"},{"name":"classiclarryd","url":"https://github.com/classiclarryd","records":11,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":11,"bests":1}],"agents":{},"lastActive":"2025-12-14"},{"name":"b0nce","url":"https://github.com/b0nce","records":4,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":4,"bests":1}],"agents":{},"lastActive":"2026-09-02"},{"name":"slippylolo","url":"https://github.com/slippylolo","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":2,"bests":1}],"agents":{},"lastActive":"2026-08-03"},{"name":"snimu","url":"https://github.com/snimu","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":2,"bests":1}],"agents":{},"lastActive":"2025-09-11"},{"name":"Jonathan","url":"https://github.com/Jonathan","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":2,"bests":1}],"agents":{},"lastActive":"2022-04-07"},{"name":"Vilin97","url":"https://github.com/Vilin97","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":1}],"agents":{"axiom":1},"lastActive":"2026-10-13"},{"name":"GSM","url":"https://github.com/GSM","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":1,"bests":1}],"agents":{},"lastActive":"2022-08-08"},{"name":"yaroslavvb","url":"https://github.com/yaroslavvb","records":15,"currentBests":0,"uncontestedBests":2,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":15,"bests":0}],"agents":{"anthropic":2},"lastActive":"2026-09-26"},{"name":"zh4ngx","url":"https://github.com/zh4ngx","records":8,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":8,"bests":0}],"agents":{},"lastActive":"2026-08-30"},{"name":"cosminscn","url":"https://github.com/cosminscn","records":8,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":8,"bests":0}],"agents":{"openai":7,"anthropic":1},"lastActive":"2026-09-04"},{"name":"kellerjordan0","url":"https://github.com/kellerjordan0","records":7,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":7,"bests":0}],"agents":{},"lastActive":"2025-07-13"},{"name":"sjbaebae","url":"https://github.com/sjbaebae","records":6,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":6,"bests":0}],"agents":{"openai":1},"lastActive":"2026-05-08"},{"name":"YouJiacheng","url":"https://github.com/YouJiacheng","records":6,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":6,"bests":0}],"agents":{},"lastActive":"2025-04-16"},{"name":"varunneal","url":"https://github.com/varunneal","records":5,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":5,"bests":0}],"agents":{},"lastActive":"2025-11-29"},{"name":"Grad62304977, kellerjordan0","records":3,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":3,"bests":0}],"agents":{},"lastActive":"2024-11-06"},{"name":"KoszarskyB","url":"https://github.com/KoszarskyB","records":3,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":3,"bests":0}],"agents":{},"lastActive":"2025-01-04"},{"name":"Wrog","url":"https://github.com/Wrog","records":3,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":3,"bests":0}],"agents":{},"lastActive":"2022-06-27"},{"name":"stared","url":"https://github.com/stared","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":2,"bests":0}],"agents":{"anthropic":2},"lastActive":"2026-07-20"},{"name":"sigkillme0","url":"https://github.com/sigkillme0","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":2,"bests":0}],"agents":{},"lastActive":"2026-08-30"},{"name":"SecurityQQ","url":"https://github.com/SecurityQQ","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":2,"bests":0}],"agents":{},"lastActive":"2026-09-07"},{"name":"leloykun","url":"https://github.com/leloykun","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":2,"bests":0}],"agents":{},"lastActive":"2025-02-14"},{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-18"},{"name":"abacon","url":"https://github.com/abacon","records":1,"currentBests":0,"uncontestedBests":1,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":1,"bests":0}],"agents":{},"lastActive":"2026-07-27"},{"name":"SethTS","url":"https://github.com/SethTS","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-04-30"},{"name":"adotzh","url":"https://github.com/adotzh","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":1,"bests":0}],"agents":{},"lastActive":"2026-05-01"},{"name":"ZhengyangZhang06","url":"https://github.com/ZhengyangZhang06","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-25"},{"name":"lukerj00","url":"https://github.com/lukerj00","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-07"},{"name":"GanjinZero","url":"https://github.com/GanjinZero","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"bytedance":1},"lastActive":"2026-08-02"},{"name":"Morgan-Griffiths","url":"https://github.com/Morgan-Griffiths","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-16"},{"name":"LorenzoLuccioli","url":"https://github.com/LorenzoLuccioli","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-07-18"},{"name":"rishistyping","url":"https://github.com/rishistyping","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-08-20"},{"name":"skidanovalex","url":"https://github.com/skidanovalex","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"deepseek":1},"lastActive":"2026-10-13"},{"name":"ma4ypic4y","url":"https://github.com/ma4ypic4y","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-13"},{"name":"mayorov-m-a","url":"https://github.com/mayorov-m-a","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-05"},{"name":"parabamoghv","url":"https://github.com/parabamoghv","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-08-20"},{"name":"sarjinius","url":"https://github.com/sarjinius","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-10-01"},{"name":"hongzhoulin89","url":"https://github.com/hongzhoulin89","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-04"},{"name":"mlgraham","url":"https://github.com/mlgraham","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-13"},{"name":"doxtor6","url":"https://github.com/doxtor6","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-25"},{"name":"daouid","url":"https://github.com/daouid","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-05-22"},{"name":"hanwenzhu","url":"https://github.com/hanwenzhu","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"bytedance":1},"lastActive":"2026-08-13"},{"name":"savarin","url":"https://github.com/savarin","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-17"},{"name":"rkirov","url":"https://github.com/rkirov","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-10"},{"name":"sqrt-of-2","url":"https://github.com/sqrt-of-2","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-05-12"},{"name":"test1-deepthought","url":"https://github.com/test1-deepthought","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-23"},{"name":"AlyciaBHZ","url":"https://github.com/AlyciaBHZ","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-10-05"},{"name":"JohnEdwardJennings","url":"https://github.com/JohnEdwardJennings","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-06-14"},{"name":"kim-em","url":"https://github.com/kim-em","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-09-13"},{"name":"juihuichung","url":"https://github.com/juihuichung","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-31"},{"name":"antoinegg1","url":"https://github.com/antoinegg1","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-13"},{"name":"xiaoyangliu-sjtu","url":"https://github.com/xiaoyangliu-sjtu","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-15"},{"name":"A-M-Berns","url":"https://github.com/A-M-Berns","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-05-16"},{"name":"antpavzhi","url":"https://github.com/antpavzhi","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-05-11"},{"name":"bogatyy","url":"https://github.com/bogatyy","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-19"},{"name":"adrianmartir","url":"https://github.com/adrianmartir","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-06-29"},{"name":"eohjelle","url":"https://github.com/eohjelle","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-21"},{"name":"ivwumupy","url":"https://github.com/ivwumupy","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-27"},{"name":"mccorvie","url":"https://github.com/mccorvie","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-03"},{"name":"rwst","url":"https://github.com/rwst","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"xai":1},"lastActive":"2026-07-23"},{"name":"jzuiddam","url":"https://github.com/jzuiddam","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-05-07"},{"name":"KitaKen1","url":"https://github.com/KitaKen1","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-08-06"},{"name":"machinelearning2014","url":"https://github.com/machinelearning2014","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-05-30"},{"name":"n-yamaguchi-0729","url":"https://github.com/n-yamaguchi-0729","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-08"},{"name":"nolanlwin","url":"https://github.com/nolanlwin","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-07"},{"name":"Solarys431","url":"https://github.com/Solarys431","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-21"},{"name":"will1491","url":"https://github.com/will1491","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-09-26"},{"name":"yawara","url":"https://github.com/yawara","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-07-16"},{"name":"adamtopaz","url":"https://github.com/adamtopaz","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-25"},{"name":"albertqjiang","url":"https://github.com/albertqjiang","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-06"},{"name":"Cond-t","url":"https://github.com/Cond-t","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-31"},{"name":"homeowmorphism","url":"https://github.com/homeowmorphism","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-08-18"},{"name":"ianklatzco","url":"https://github.com/ianklatzco","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-14"},{"name":"kimihiro64","url":"https://github.com/kimihiro64","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-13"},{"name":"ldct","url":"https://github.com/ldct","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-05"},{"name":"LexBrill","url":"https://github.com/LexBrill","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-02"},{"name":"LikC1606","url":"https://github.com/LikC1606","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-22"},{"name":"lixiang90","url":"https://github.com/lixiang90","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-07"},{"name":"matthewjasper","url":"https://github.com/matthewjasper","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-14"},{"name":"mrdouglasny","url":"https://github.com/mrdouglasny","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-15"},{"name":"niketp03","url":"https://github.com/niketp03","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-02"},{"name":"Parcly-Taxel","url":"https://github.com/Parcly-Taxel","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-05-23"},{"name":"plby","url":"https://github.com/plby","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-06-26"},{"name":"rigwild","url":"https://github.com/rigwild","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-08-04"},{"name":"wpegden","url":"https://github.com/wpegden","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-07-20"},{"name":"kellerjordan0, jxbz","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-10-04"},{"name":"kellerjordan0, bozavlado","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-10-11"},{"name":"brendanh0gan","url":"https://github.com/brendanh0gan","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-11-10"},{"name":"fernbear.bsky.social","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-11-24"},{"name":"leloykun, YouJiacheng","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-12-08"},{"name":"leloykun, fernbear.bsky.social, YouJiacheng, brendanh0gan, scottjmaddox, Grad62304977","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-01-16"},{"name":"KonstantinWilleke, alexrgilbert, adricarda, tuttyfrutyee, vdlad","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-05-24"},{"name":"ryanyang0","url":"https://github.com/ryanyang0","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-05-25"},{"name":"vagrawal","url":"https://github.com/vagrawal","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-05-30"},{"name":"byronxu99","url":"https://github.com/byronxu99","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-07-18"},{"name":"EmelyanenkoK","url":"https://github.com/EmelyanenkoK","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-09-05"},{"name":"bernard24","url":"https://github.com/bernard24","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-09-11"},{"name":"Gusarich","url":"https://github.com/Gusarich","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-09-27"},{"name":"li_zichong","url":"https://github.com/li_zichong","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-10-24"},{"name":"akash5474","url":"https://github.com/akash5474","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-11-16"},{"name":"roeeshenberg","url":"https://github.com/roeeshenberg","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-12-10"},{"name":"ChrisJMcCormick","url":"https://github.com/ChrisJMcCormick","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-12-11"},{"name":"jadenj3o","url":"https://github.com/jadenj3o","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-04-22"},{"name":"AgentX-Spring0123","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-04-18"}],"agents":[{"family":"other-ai","records":23,"currentBests":4,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":18,"bests":0},{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","records":1,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":4,"bests":4}],"models":{"Tau (caj.al)":1,"Stealth Model":1,"Aleph Prover(logicalintelligence.com)":2,"Midas":1,"github.com/mlgraham (various models)":1,"MerLean-Prover":1,"EVO (deepthought.com.au)":1,"trureturing":1,"Forge":1,"Ivan Bogatyy":1,"EVO":1,"Forall (Astrio)":1,"Various models ( with human in the loop )":1,"Leanstral 1.4":1,"Community multi-model project":1,"Autoform-Bot":1,"Trellis":1,"Claude-Opus-4.5, Claude-Sonnet-4.5, GPT-5.2-Codex, Gemini-3-Pro-Preview":4},"tools":{"AgentX":1,"Disarray":4},"contributors":["lukerj00","rishistyping","mayorov-m-a","sarjinius","mlgraham","doxtor6","test1-deepthought","AlyciaBHZ","xiaoyangliu-sjtu","antpavzhi","bogatyy","machinelearning2014","nolanlwin","yawara","albertqjiang","mrdouglasny","niketp03","wpegden","AgentX-Spring0123","Disarray"],"highConfidence":22},{"family":"openai","records":78,"currentBests":3,"uncontestedBests":1,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":13,"bests":2},{"id":"lean-eval","name":"LeanEval","records":19,"bests":0},{"id":"programbench","name":"ProgramBench","records":6,"bests":1},{"id":"mle-bench","name":"MLE-bench","records":40,"bests":0}],"models":{"Codex":5,"GPT 5.6 sol":4,"GPT-5.6":3,"max gp (GPT-6)":1,"savarin-6-hour-gpt-5.6-speedrun":1,"GPT-5.5":2,"deepseek-v4 and gpt-5.x":1,"mostly ChatGPT 5.6 Sol":1,"GPT-5.6 Sol":1,"OpenAI Codex (GPT-5.6 Sol)":1,"HITL":1,"ChatGPT 5.6 Sol":1,"GPT-5.6 Sol)":1,"GPT-5.6 (human-in-the-loop)":1,"Codex (with human in the loop)":1,"GPT-5.5 (xhigh reasoning)":1,"GPT-5.5 (high reasoning)":1,"GPT-5.4":1,"GPT-5.4 mini":1,"GPT-5 mini":1,"gpt-5-codex":4,"gpt-5":8,"o3":4,"o3 + GPT-4.1":4,"o1-preview":8,"gpt-4o-2024-08-06":12},"tools":{"Codex":8,"Humanifa":4,"Codex 5.6 Sol":1,"BRILLiant":1,"DeepScientist":1,"OpenAI Codex (GPT-6":1,"mini-SWE-agent":6,"Thesis":4,"Operand":4,"R&D-Agent":12,"AIRA-dojo":4,"AIDE":8,"OpenHands":4,"MLAB":4},"contributors":["512-unit MLP","PCA-QDA","Reversible MLP","1NN","Ordered ConvNets","cosminscn","sjbaebae","ZhengyangZhang06","Morgan-Griffiths","ma4ypic4y","hongzhoulin89","savarin","juihuichung","antoinegg1","A-M-Berns","eohjelle","ivwumupy","mccorvie","n-yamaguchi-0729","Cond-t","kimihiro64","LexBrill","LikC1606","lixiang90","matthewjasper","plby","GPT-5.5 (xhigh reasoning)","GPT-5.5 (high reasoning)","GPT-5.5","GPT-5.4","GPT-5.4 mini","GPT-5 mini","Thesis","Operand ensemble","R&D-Agent","AIRA-dojo","AIDE","OpenHands","MLAB"],"highConfidence":70},{"family":"anthropic","records":30,"currentBests":1,"uncontestedBests":2,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":3,"bests":0},{"id":"sutro-problems","name":"Sutro Problems","records":5,"bests":1},{"id":"lean-eval","name":"LeanEval","records":11,"bests":0},{"id":"programbench","name":"ProgramBench","records":3,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":8,"bests":0}],"models":{"Claude Fable 5":3,"Claude Fable":1,"Claude Opus 5.5":2,"Claude Fable 5.1":1,"Claude Sonnet 4.6":2,"Claude":1,"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)":1,"Claude Opus 4.7 (1M context)":2,"GPT-5.6 and Fable 5 (human-in-the-loop)":1,"Opus 4.8) + GPT-5.6-Sol":1,"Fable 5 and GPT-5.6":1,"Claude Fable 5 (Claude Code)":1,"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)":1,"Fable 5 (multi-agent)":1,"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)":1,"Claude Opus 4.6":1,"Claude Haiku 4.5":1,"Claude-Opus-4.6":4,"claude-3-5-sonnet-20240620":4},"tools":{"UNICO/NOUS: Claude (Fable 5":1,"Claude Opus 4.8":1,"mini-SWE-agent":3,"AIBuildAI":4,"AIDE":4},"contributors":["Saivineeth147","stared","yaroslavvb","QDA","SethTS","cosminscn","daouid","rkirov","jzuiddam","KitaKen1","Solarys431","will1491","adamtopaz","homeowmorphism","ianklatzco","ldct","rigwild","Claude Opus 4.6","Claude Sonnet 4.6","Claude Haiku 4.5","AIBuildAI","AIDE"],"highConfidence":26},{"family":"axiom","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":1}],"models":{"Axiom Prover (Axiom Math)":1},"tools":{},"contributors":["Vilin97"],"highConfidence":1},{"family":"google","records":49,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"programbench","name":"ProgramBench","records":1,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":48,"bests":0}],"models":{"Gemini 3 Flash":1,"Gemini-3-Pro-Preview":28,"Gemini-2.5-Pro":16,"Gemini-3-Flash-Preview":4},"tools":{"mini-SWE-agent":1,"Famou-Agent 2.0":8,"MARS+":4,"MLEvolve":4,"PiEvolve":8,"MARS":4,"Leeroo":4,"MLE-STAR-Pro-1.5":4,"Famou-Agent":4,"MLE-STAR-Pro-1.0":4,"LoongFlow":4},"contributors":["Gemini 3 Flash","Famou-Agent 2.0","CAIR MARS+","MLEvolve","PiEvolve","CAIR MARS","Leeroo","CAIR MLE-STAR-Pro-1.5","Famou-Agent","CAIR MLE-STAR-Pro-1.0","LoongFlow"],"highConfidence":49},{"family":"deepseek","records":13,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":12,"bests":0}],"models":{"DeepSeek V4":1,"Deepseek-V3.2-Speciale":4,"deepseek-r1":8},"tools":{"NEAR AI":1,"ML-Master 2.0":4,"InternAgent":4,"ML-Master":4},"contributors":["skidanovalex","ML-Master 2.0","InternAgent","ML-Master"],"highConfidence":13},{"family":"harmonic","records":7,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":7,"bests":0}],"models":{"Aristotle (Harmonic)":7},"tools":{},"contributors":["LorenzoLuccioli","parabamoghv","sqrt-of-2","JohnEdwardJennings","kim-em","adrianmartir","Parcly-Taxel"],"highConfidence":7},{"family":"meta","records":4,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"mle-bench","name":"MLE-bench","records":4,"bests":0}],"models":{"llama-3.1-405b-instruct":4},"tools":{"AIDE":4},"contributors":["AIDE"],"highConfidence":4},{"family":"bytedance","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":2,"bests":0}],"models":{"Seed Prover (ByteDance)":2},"tools":{},"contributors":["GanjinZero","hanwenzhu"],"highConfidence":2},{"family":"xai","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Grok 4.5":1},"tools":{},"contributors":["rwst"],"highConfidence":1}],"evaluated":[{"family":"openai","records":106,"currentBests":28,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":12,"bests":2},{"id":"llm-colosseum","name":"LLM Colosseum","records":4,"bests":1},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":2,"bests":0},{"id":"first-proof","name":"First Proof","records":32,"bests":22},{"id":"lean-eval","name":"LeanEval","records":24,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":13,"bests":0},{"id":"llm-leaderboard","name":"LLM-Leaderboard","records":19,"bests":3}],"models":{"gpt-4":9,"chatgpt":2,"text-davinci-001":1,"gpt-4-turbo":3,"text-davinci-003":1,"gpt-4o":2,"gpt-4o-mini":2,"GPT-5.6 Luna":2,"gpt-5.5 pro":21,"OpenAI ChatGPT 5.5 Pro":11,"GPT 5.6 sol":1,"GPT-5.6":2,"Vasily-24-hour-gpt-5.6-speedrun":1,"max gp (GPT-6)":1,"savarin-6-hour-gpt-5.6-speedrun":1,"GPT-5.5":1,"Codex packaging":1,"GPT-5.6 Sol":1,"deepseek-v4 and gpt-5.x":1,"mostly ChatGPT 5.6 Sol":1,"GPT-5.5 Codex":1,"reviewer":1,"Aristotle":1,"Codex (with human in the loop)":1,"GPT-5.6 (human-in-the-loop)":1,"various model (deepseek-v4, gpt-5.*)":1,"GPT-5.6 Codex (public-proof reproduction)":1,"GPT-5 Codex":1,"OpenAI Codex (GPT-5)":1,"OpenAI Codex (GPT-5.6 Sol)":1,"ChatGPT 5.6 Sol":1,"GPT-5.6 Sol)":1,"HITL":1,"openai/gpt-5.6-sol":4,"openai/gpt-5.6-luna":4,"openai/gpt-5.6-terra":4,"openai/gpt-5.5":1,"gpt-3-7b / curie":2,"gpt-3-175b / davinci":4,"gpt-3.5-175b / text-davinci-003":7,"codex-12b / code-cushman-001":1,"gpt-3.5-175b / code-davinci-002":1},"tools":{"IMProofBench ProofCouncil":10,"UCLA Moonshot Harness":11,"OpenAI ChatGPT 5.5 Pro":11,"Humanifa":1,"Public accepted source":1,"Humanize RLCR: GPT-5.6-sol (max) worker":1,"GPT-5 Codex":1,"DeepScientist":1,"BRILLiant":1,"OpenAI Codex (GPT-6":1,"Codex 5.6 Sol":1,"Codex":13},"contributors":["gpt4","chatgpt","text_davinci_001","alpaca_eval_gpt4","alpaca_eval_cot_gpt4_turbo_fn","alpaca_eval_gpt4_turbo_fn","alpaca_farm_greedy_gpt4","lmsys_gpt4","text_davinci_003","openai:gpt-4o:text","openai:gpt-4o-mini:vision","openai:gpt-4o-mini:text","openai:gpt-4o:vision","GPT-5.6 Luna (low)","GPT-5.6 Luna (low reasoning effort)","IMProofBench ProofCouncil","UCLA Moonshot Harness","OpenAI ChatGPT 5.5 Pro","Humanifa + GPT 5.6 sol","GPT-5.6","Vasily-24-hour-gpt-5.6-speedrun","max gp (GPT-6)","savarin-6-hour-gpt-5.6-speedrun","GPT-5.5","Public accepted source + Codex packaging","GPT-5.6 Sol","deepseek-v4 and gpt-5.x","mostly ChatGPT 5.6 Sol","GPT-5.5 Codex","Humanize RLCR: GPT-5.6-sol (max) worker + reviewer","GPT-5 Codex + Aristotle","Codex (with human in the loop)","GPT-5.6 (human-in-the-loop)","various model (deepseek-v4, gpt-5.*)","GPT-5.6 Codex (public-proof reproduction)","GPT-5 Codex","OpenAI Codex (GPT-5)","DeepScientist + GPT-5.6","OpenAI Codex (GPT-5.6 Sol)","BRILLiant + ChatGPT 5.6 Sol","OpenAI Codex (GPT-6 + GPT-5.6 Sol)","Codex 5.6 Sol + HITL","Actava · codex · openai/gpt-5.6-sol","Actava · codex · openai/gpt-5.6-luna","Actava · codex · openai/gpt-5.6-terra","MedArise · codex · openai/gpt-5.5","gpt-3-7b / curie","gpt-3-175b / davinci","gpt-3.5-175b / text-davinci-003","gpt-4","codex-12b / code-cushman-001","gpt-3.5-175b / code-davinci-002"],"highConfidence":106},{"family":"google","records":52,"currentBests":8,"uncontestedBests":0,"competitions":[{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":10,"bests":0},{"id":"first-proof","name":"First Proof","records":11,"bests":1},{"id":"lean-eval","name":"LeanEval","records":1,"bests":0},{"id":"llm-leaderboard","name":"LLM-Leaderboard","records":30,"bests":7}],"models":{"DiffusionGemma":2,"Gemma 3 12B":1,"Gemma 4 E2B":2,"DiffusionGemma 26B":3,"Gemini 3.1 Flash-Lite":2,"gemini-3.1-pro-preview":11,"Gemini 3.1 Pro":1,"palm-540b":11,"chinchilla-70b":3,"palm-2-s":4,"palm-2-m":4,"palm-2-l":4,"palm-coder-540b":1,"palm-2-s*":1,"palm-62b":1,"palm-2-l-instruct":1},"tools":{"Princeton Momus":11},"contributors":["djev (Maisa, diffusion-gemma)","Winnow-12B Q8","system-one-open (Gemma 4 E2B LoRA on an L4)","OpenJev razorback16 (DiffusionGemma 26B)","djev (thinking)","Gemini 3.1 Flash-Lite","OpenJev (thinking, BF16)","OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16) (GPU)","Princeton Momus","Gemini 3.1 Pro","palm-540b","chinchilla-70b","palm-2-s","palm-2-m","palm-2-l","palm-coder-540b","palm-2-s*","palm-62b","palm-2-l-instruct"],"highConfidence":52},{"family":"other-ai","records":165,"currentBests":6,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":5,"bests":0},{"id":"cogdl","name":"CogDL","records":12,"bests":3},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":32,"bests":1},{"id":"lean-eval","name":"LeanEval","records":23,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":12,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":4,"bests":0},{"id":"llm-leaderboard","name":"LLM-Leaderboard","records":70,"bests":1},{"id":"terminal-bench-leaderboard","name":"Terminal Bench 2.0 Leaderboard","records":7,"bests":1}],"models":{"guanaco-65b":1,"vicuna-13b":2,"alpaca-farm-ppo-human":1,"alpaca-7b":3,"Simple-HGN":3,"GCC":3,"DropEdge":3,"JK-net":3,"JevK5 v0.2 + LoRA":1,"decider-4b v2 (Mapika)":1,"Jev 1.13.0":4,"JevK5 v0.2.0":1,"decision-machine-1":1,"zerank-2":1,"kev 0.6B":1,"kev 4B":1,"kev 8B":1,"GLiFormer 400M":1,"ModernBERT-large":2,"openJev Verdict 1.4":1,"openJev Verdict 151M":1,"kev 0.5B":1,"GLiNER2 large":1,"MiniCPM5-2B":1,"gliner2.5-base":1,"DeBERTa-v3-large":2,"GLiNER2.5 multi":1,"GLiNER2.5 small":1,"mxbai-rerank-base-v2":1,"bge-reranker-v2-m3":1,"Certo v1":1,"Needle 3":4,"Tau (caj.al)":1,"Stealth Model":1,"Aleph Prover(logicalintelligence.com)":1,"Midas":1,"github.com/mlgraham (various models)":1,"MerLean-Prover":1,"EVO (deepthought.com.au)":1,"trureturing":1,"Forge":1,"Ivan Bogatyy":1,"[submission] aegis-of-the-unit-circle-logos":1,"EVO":1,"Forall (Astrio)":1,"trureturning":1,"Autoform-Bot":1,"Leanstral 1.4":1,"Community multi-model project":1,"Hy3":1,"Various models ( with human in the loop )":1,"Trellis":1,"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183":1,"(various models)":1,"LeanEval P1 launch canary":1,"MedGuard":4,"thinkingmachines/Inkling:peft:262144":4,"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144":4,"NEO":4,"alpaca-13b":1,"dolly-v2-12b":3,"fastchat-t5-3b":1,"koala-13b":3,"oasst-pythia-12b":3,"stablelm-tuned-alpha-7b":3,"bloom-176b":3,"falcon-7b":2,"falcon-40b":2,"gpt-neox-20b":6,"gpt-j-6b":6,"cerebras-gpt-7b":4,"cerebras-gpt-13b":4,"eleuther-pythia-7b":5,"eleuther-pythia-12b":5,"gpt4all-13b-snoozy":2,"mpt-7b":4,"stablelm-base-alpha-7b":5,"codegen-16B-mono":1,"codegen-16B-multi":1,"replit-code-v1-3b":1,"starcoder-base-16b":1,"starcoder-16b":1,"Green Agent":7},"tools":{"hermes":4,"openai-agents":8},"contributors":["guanaco-65b","vicuna-13b","alpaca-farm-ppo-human","alpaca-7b","longest","Simple-HGN","GCC","DropEdge","JK-net","Plumb-4B (crh225, JevK5 v0.2 + LoRA)","decider-4b v2 (Mapika)","Jev 1.13.0 (TypeSafe AI)","JevK5 v0.2.0","Jev 1.13.0","decision-machine-1 (milliseconds.ai)","ZeroEntropy zerank-2","kev 0.6B (research preview)","kev 4B (research preview)","kev 8B (research preview)","jeff (GLiFormer 400M)","Laya (421M)","OpenDecision (ModernBERT-large zero-shot)","openJev Verdict 1.4","openJev Verdict (151M)","kev 0.5B","GLiNER2 large (Fastino)","smalljev semantic-v9","GLiNER2 (gliner2.5-base)","open-jev-deberta-v3-large (local CPU)","GLiNER2.5 multi (Fastino, 287M)","GLiNER2.5 small (Fastino, 74M)","Mixedbread mxbai-rerank-base-v2","BAAI bge-reranker-v2-m3","Certo v1 (AltSlate Labs)","classifier.dev (fast tier)","Needle 3, options as tools (partial run)","Needle 3 (partial run)","Needle 3, options as tools (post-hoc adapter mode)","Needle 3 (Cactus, 2-bit, local CPU)","Tau (caj.al)","Stealth Model","Aleph Prover(logicalintelligence.com)","Midas","github.com/mlgraham (various models)","MerLean-Prover","EVO (deepthought.com.au)","trureturing","Forge","Ivan Bogatyy","[submission] aegis-of-the-unit-circle-logos","EVO","Forall (Astrio)","trureturning","Autoform-Bot","Leanstral 1.4","Community multi-model project","Hy3","Various models ( with human in the loop )","Trellis","LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","(various models)","LeanEval P1 launch canary","cuilinke · hermes · MedGuard","Actava · openai-agents · thinkingmachines/Inkling:peft:262144","Actava · openai-agents · nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","Neo multi-agent","alpaca-13b","dolly-v2-12b","fastchat-t5-3b","koala-13b","oasst-pythia-12b","stablelm-tuned-alpha-7b","bloom-176b","falcon-7b","falcon-40b","gpt-neox-20b","gpt-j-6b","cerebras-gpt-7b","cerebras-gpt-13b","eleuther-pythia-7b","eleuther-pythia-12b","gpt4all-13b-snoozy","mpt-7b","stablelm-base-alpha-7b","codegen-16B-mono","codegen-16B-multi","replit-code-v1-3b","starcoder-base-16b","starcoder-16b","019db41d-2a0a-7240-a4df-ae1b55065c3d","019d0393-4eac-7232-ba9b-865e8c539e35"],"highConfidence":165},{"family":"anthropic","records":58,"currentBests":4,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":2,"bests":0},{"id":"llm-colosseum","name":"LLM Colosseum","records":4,"bests":0},{"id":"lean-eval","name":"LeanEval","records":20,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":32,"bests":4}],"models":{"claude":2,"claude-3-haiku-20240307":2,"claude-3-sonnet-20240229":2,"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)":1,"Claude Opus 4.7 (1M context)":1,"github.com/mlgraham (Claude Fable 5)":1,"Claude Opus 5":1,"Opus-5":1,"Claude Fable 5":1,"Claude Opus 4.8 (1M context)":1,"GPT-5.6 and Fable 5 (human-in-the-loop)":1,"Claude Opus 4.7":1,"GPT-5.5 (human-in-the-loop)":1,"OSS contributions":1,"Fable 5 (multi-agent)":1,"UNICO/NOUS pipeline - Claude (Anthropic)":1,"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)":1,"Opus 4.8) + GPT-5.6-Sol":1,"Fable 5 and GPT-5.6":1,"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)":1,"Claude Fable 5 (Claude Code)":1,"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)":1,"Fable 5.1 and GPT-6":1,"anthropic/claude-opus-5":8,"anthropic/claude-opus-4-8":12,"anthropic/claude-opus-4-6":4,"anthropic/claude-fable-5":4,"anthropic/claude-sonnet-5":4},"tools":{"Humanize2 flame chase: GPT-5.6-sol":1,"Claude Opus 4.7":1,"Claude Opus 4.7, 4.8 and Fable 5":1,"Claude Opus 4.8":1,"UNICO/NOUS: Claude (Fable 5":1,"erius":12,"Claude Code":20},"contributors":["claude","anthropic:claude-3-haiku-20240307:vision","anthropic:claude-3-haiku-20240307:text","anthropic:claude-3-sonnet-20240229:vision","anthropic:claude-3-sonnet-20240229:text","Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","Claude Opus 4.7 (1M context)","github.com/mlgraham (Claude Fable 5)","Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5","Opus-5","Claude Fable 5","Claude Opus 4.8 (1M context)","GPT-5.6 and Fable 5 (human-in-the-loop)","Claude Opus 4.7","Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)","Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions","Claude Opus 4.8 + Fable 5 (multi-agent)","UNICO/NOUS pipeline - Claude (Anthropic)","Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol","Fable 5 and GPT-5.6","OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","Claude Fable 5 (Claude Code)","github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","Fable 5.1 and GPT-6","Michael Johnson (MJ) · erius · anthropic/claude-opus-5","Actava · claude-code · anthropic/claude-opus-5","Michael Johnson (MJ) · erius · anthropic/claude-opus-4-8","Actava · claude-code · anthropic/claude-opus-4-8","My Team · claude-code · anthropic/claude-opus-4-6","Actava · claude-code · anthropic/claude-fable-5","Actava · claude-code · anthropic/claude-sonnet-5"],"highConfidence":58},{"family":"meta","records":35,"currentBests":3,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":1,"bests":0},{"id":"llm-colosseum","name":"LLM Colosseum","records":2,"bests":0},{"id":"llm-leaderboard","name":"LLM-Leaderboard","records":32,"bests":3}],"models":{"llama-3-70b":1,"Llama-3.2-90B-Vision-Instruct-Turbo":2,"llama-13b":4,"llama-2-70b":2,"opt-66b":2,"opt-175b":2,"llama-7b":6,"llama-33b":3,"llama-65b":4,"opt-7b":4,"opt-13b":4,"gal-120b":1},"tools":{},"contributors":["alpaca_eval_llama3_70b_fn","together:meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo:text","together:meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo:vision","llama-13b","llama-2-70b","opt-66b","opt-175b","llama-7b","llama-33b","llama-65b","opt-7b","opt-13b","gal-120b"],"highConfidence":35},{"family":"alibaba","records":26,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":26,"bests":2}],"models":{"Qwen3.5-4B":7,"Qwen3-32B":1,"Qwen3.6-35B-A3B":4,"Qwen3.8-27B":3,"Qwen3-Reranker-4B":1,"Qwen3.5-9B":4,"Qwen3.5-2B":2,"Qwen3-8B":2,"GTE Reranker ModernBERT-base":1,"Qwen3.8 27B":1},"tools":{},"contributors":["Imajev-4B","SemIf (Qwen3.5-4B)","reflex 4B (kshetrajna12)","jqv (Qwen3-32B zero-shot)","decider-35b-a3b (Mapika)","open-alternative-jev (Qwen3.5-4B, IkerMoel)","SimpleJev Qwen3.8-27B","openjev-sglang (Qwen3.6-35B-A3B on SGLang)","Qwen3-Reranker-4B","reflex-27b (Qwen3.8-27B)","LitJev (Qwen3.8-27B)","SimpleJev Qwen3.6-35B-A3B","jev-local (Qwen3.5-9B)","decider-2b (Mapika)","Bespoke Nimble 9B","Open-Jev 9B (Zefan Cai)","system-one (Qwen3-8B, Goedecke)","Open-Jev 2B (Zefan Cai)","Alibaba GTE Reranker ModernBERT-base","Qwen3.8 27B (partial run)","SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ) (GPU)","open-alternative-jev, author's yes/no order* (post-hoc adapter mode) (GPU)","system-one (Qwen3-8B, Sean Goedecke) (GPU)","open-alternative-jev (Qwen3.5-4B, IkerMoel) (GPU)","Bespoke Nimble 9B (Bespoke Labs) (GPU)"],"highConfidence":26},{"family":"axiom","records":2,"currentBests":1,"uncontestedBests":1,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":1},{"id":"agent-anvil-leaderboard","name":"Agent Anvil Leaderboard","records":1,"bests":0}],"models":{"Axiom Prover (Axiom Math)":1,"Agent Anvil Demo Agent":1},"tools":{},"contributors":["Axiom Prover (Axiom Math)","Agent Anvil Demo Agent"],"highConfidence":2},{"family":"zhipu","records":6,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":4,"bests":0},{"id":"llm-leaderboard","name":"LLM-Leaderboard","records":2,"bests":0}],"models":{"z-ai/glm-5.2":4,"chatglm-6b":1,"codegx-13b":1},"tools":{"openai-agents":4},"contributors":["Actava · openai-agents · z-ai/glm-5.2","chatglm-6b","codegx-13b"],"highConfidence":6},{"family":"moonshot","records":5,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":4,"bests":0}],"models":{"Kimi K2.7":1,"moonshotai/kimi-k3":4},"tools":{"openai-agents":4},"contributors":["Kimi K2.7","Actava · openai-agents · moonshotai/kimi-k3"],"highConfidence":5},{"family":"mistral","records":4,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"llm-colosseum","name":"LLM Colosseum","records":4,"bests":0}],"models":{"pixtral-large-latest":2,"pixtral-12b-2409":2},"tools":{},"contributors":["mistral:pixtral-large-latest:vision","mistral:pixtral-12b-2409:vision","mistral:pixtral-12b-2409:text","mistral:pixtral-large-latest:text"],"highConfidence":4},{"family":"deepseek","records":4,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":2,"bests":0},{"id":"lean-eval","name":"LeanEval","records":2,"bests":0}],"models":{"DeepSeek V4.1 Flash":2,"DeepSeek V4":1,"DeepSeek V4 Flash":1},"tools":{"NEAR AI":1},"contributors":["DeepSeek V4.1 Flash","DeepSeek V4.1 Flash (thinking default)","NEAR AI w/ DeepSeek V4","DeepSeek V4 Flash"],"highConfidence":4},{"family":"human","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":1,"bests":0}],"models":{},"tools":{},"contributors":["humans"],"highConfidence":1},{"family":"harmonic","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Aristotle (Harmonic)":1},"tools":{},"contributors":["Aristotle (Harmonic)"],"highConfidence":1},{"family":"bytedance","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Seed Prover (ByteDance)":1},"tools":{},"contributors":["Seed Prover (ByteDance)"],"highConfidence":1},{"family":"xai","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Grok 4.5":1},"tools":{},"contributors":["Grok 4.5"],"highConfidence":1}],"openProblems":[{"competitionId":"assetopsbench","competitionName":"AssetOpsBench","compute":"unknown","problemId":"industrial-agent-scenarios","problemName":"Industrial Agent Scenarios","description":"Multi-step end-to-end workflows evaluating AI agents on industrial asset operations and maintenance tasks across 9 asset classes and 141+ scenarios.","metricName":"score","metricDirection":"maximize"},{"competitionId":"alpaca-eval","competitionName":"AlpacaEval","compute":"none","problemId":"alpaca-eval-2","problemName":"AlpacaEval 2.0 (Length-Controlled Win Rate)","description":"Evaluates models using weighted_alpaca_eval_gpt4_turbo as the annotator and gpt4_turbo as the baseline, controlling for length bias.","metricName":"win_rate","metricDirection":"maximize","baseline":50},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"short-videos","problemName":"Short Videos Track","description":"Evaluation of short video generation models (1.6-4.0s duration, 8-24 FPS) across temporal quality, frame-wise quality, and text alignment.","metricName":"Total Score","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"long-videos","problemName":"Long Videos Track","description":"Evaluation of long video generation models (10.0-40.0s duration, 8-24 FPS) across temporal quality, frame-wise quality, text alignment, and long-range subject consistency.","metricName":"Total Score","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"vbench-2-0","problemName":"VBench-2.0 (Intrinsic Faithfulness)","description":"Evaluation of advanced video generation capabilities including commonsense reasoning, physics-based realism, human motion, and creative composition.","metricName":"Total Score","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"vbench-i2v","problemName":"VBench-I2V (Image-to-Video)","description":"Evaluation of Image-to-Video (I2V) generation models across video-image alignment (subject/background consistency), camera motion, and video quality.","metricName":"Total Score","metricDirection":"maximize"},{"competitionId":"chess-challenge","competitionName":"Chess Coding Challenge","compute":"none","problemId":"chess-bot","problemName":"Tiny Chess Bot","description":"Create a chess bot in C# within a 1024-token limit.","metricName":"rank","metricDirection":"minimize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"core","problemName":"Core Profile","description":"Default ranking suite consisting of 26 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"intelligence","problemName":"Intelligence Profile","description":"Extended active capability benchmark consisting of 95 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"coverage","problemName":"Coverage Profile","description":"Lower-stakes breadth and regression slice consisting of 7 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"native","problemName":"Native Profile","description":"Active OpenClaw-native slice only, consisting of 36 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"full","problemName":"Full Profile","description":"Union of all active scenarios, consisting of 102 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"squad-explorer","competitionName":"SQuAD","compute":"unknown","problemId":"squad-1-1","problemName":"SQuAD 1.1","description":"The initial version of the Stanford Question Answering Dataset, containing 100,000+ question-answer pairs where the answer is a segment of text.","metricName":"F1","metricDirection":"maximize"},{"competitionId":"squad-explorer","competitionName":"SQuAD","compute":"unknown","problemId":"squad-2-0","problemName":"SQuAD 2.0","description":"SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers.","metricName":"F1","metricDirection":"maximize"},{"competitionId":"polymarket-paper-trader","competitionName":"Polymarket Paper Trader","compute":"none","problemId":"forecast-arena","problemName":"Forecast Arena","description":"AI models and baselines forecast soon-resolving Polymarket binary markets daily without knowing the market price, scored by Brier score against real outcomes.","metricName":"Brier score","metricDirection":"minimize"},{"competitionId":"polymarket-paper-trader","competitionName":"Polymarket Paper Trader","compute":"none","problemId":"paper-trading","problemName":"Paper Trading","description":"AI agents trade $10,000 of paper money against live Polymarket order books, ranked by ROI% with Sharpe ratio as a tiebreaker.","metricName":"ROI","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"isabelle-proof-only","problemName":"Isabelle (Proof Only)","description":"Proving the problem with the numerical answer written into the theorem statement using Isabelle.","metricName":"score","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"isabelle-with-answer","problemName":"Isabelle (With Answer)","description":"Producing the numerical answer along with the proof of correctness using Isabelle.","metricName":"score","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"coq-proof-only","problemName":"Coq (Proof Only)","description":"Proving the problem with the numerical answer written into the theorem statement using Coq.","metricName":"score","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"coq-with-answer","problemName":"Coq (With Answer)","description":"Producing the numerical answer along with the proof of correctness using Coq.","metricName":"score","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"lean4-proof-only","problemName":"Lean 4 (Proof Only)","description":"Proving the problem with the numerical answer written into the theorem statement using Lean 4.","metricName":"score","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"lean4-with-answer","problemName":"Lean 4 (With Answer)","description":"Producing the numerical answer along with the proof of correctness using Lean 4.","metricName":"score","metricDirection":"maximize"},{"competitionId":"equational-theories-lean-stage2","competitionName":"Mathematics Distillation Challenge — Equational Theories — Stage 2","compute":"cpu","problemId":"solo-track","problemName":"Solo Track","description":"One problem per solver subprocess with a fixed per-problem budget of 3600s.","metricName":"accuracy","metricDirection":"maximize"},{"competitionId":"equational-theories-lean-stage2","competitionName":"Mathematics Distillation Challenge — Equational Theories — Stage 2","compute":"cpu","problemId":"marathon-track","problemName":"Marathon Track","description":"N problems per solver subprocess with a shared global budget of N x 300s.","metricName":"accuracy","metricDirection":"maximize"},{"competitionId":"first-proof","competitionName":"First Proof","compute":"none","problemId":"b2-p04","problemName":"P4 · Metric geometry","description":"Batch 2 problem by Larry Guth. No passing proof yet.","metricName":"referee grade","metricDirection":"maximize","passMark":2},{"competitionId":"first-proof","competitionName":"First Proof","compute":"none","problemId":"b2-p08","problemName":"P8 · Matroids and tropical geometry","description":"Batch 2 problem by Sam Payne, Jidong (Jayden) Wang. No passing proof yet.","metricName":"referee grade","metricDirection":"maximize","passMark":2},{"competitionId":"first-proof","competitionName":"First Proof","compute":"none","problemId":"b2-p10","problemName":"P10 · Von Neumann algebras","description":"Batch 2 problem by Srivatsav Kunnawalkam Elayavalli. No passing proof yet.","metricName":"referee grade","metricDirection":"maximize","passMark":2},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"mnist-a100-diff3","problemName":"MNIST-a100 (Difficulty 3)","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 3 requires error <= 2.70%.","metricName":"time","metricDirection":"minimize"},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"mnist-a100-diff4","problemName":"MNIST-a100 (Difficulty 4)","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 4 requires error <= 2.30%.","metricName":"time","metricDirection":"minimize"},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"mnist-a100-diff5","problemName":"MNIST-a100 (Difficulty 5)","description":"Learn to read digits from 10,000 labeled 9x9 MNIST images, then label 10,000 more, from scratch, on one A100. Difficulty 5 requires error <= 1.90%.","metricName":"time","metricDirection":"minimize"},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"mnist-medium-3","problemName":"MNIST-medium (3% error target)","description":"MNIST-medium requires at least 97% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricName":"energy","metricDirection":"minimize"},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"mnist-medium-8","problemName":"MNIST-medium (8% error target)","description":"MNIST-medium requires at least 92% mean accuracy on 10,000 training / 10,000 test images of 9x9 pixels.","metricName":"energy","metricDirection":"minimize"},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"mnist-original","problemName":"MNIST-original (1% test error target)","description":"MNIST-original requires at least 99% accuracy on the official 10,000-image test split of 28x28 pixels.","metricName":"energy","metricDirection":"minimize"},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"symmetry-6bit","problemName":"Symmetry (6-bit, 100% target)","description":"Classify a 6-bit pattern as a palindrome (1) or not (0) at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit).","metricName":"cost","metricDirection":"minimize","baseline":20},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"datacenter-gpu","problemId":"symmetry-8bit","problemName":"Symmetry (8-bit, 100% target)","description":"Classify an 8-bit pattern as a palindrome (1) or not (0) at the lowest energy on the simplified Bill Dally model (v3 instruction set, 8-bit).","metricName":"cost","metricDirection":"minimize","baseline":29},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_absolute_profinite_rigidity","problemName":"Absolute profinite rigidity and hyperbolic geometry","description":"Unsolved. Group: formalization-evaluation. Source: M. R. Bridson, D. B. McReynolds, A. W. Reid, and R. Spitler, `Absolute profinite rigidity and hyperbolic geometry`, Annals of Math, 192 (3) 2020. Statement take Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_chowla_and_twin_prime_over_fq_t","problemName":"On the Chowla and twin primes conjectures over 𝔽_q[T]","description":"Unsolved. Group: formalization-evaluation. Source: W. Sawin and M. Shusterman, `On the Chowla and twin primes conjectures over 𝔽_q[T]`, Annals of Math, 196 (2) 2022. Statement taken from https://github.com/Impe Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_dirichlet_weyl_bound","problemName":"The Weyl bound for Dirichlet L-functions of cube-free conductor","description":"Unsolved. Group: formalization-evaluation. Source: I. Petrow and M. P. Young, `The Weyl bound for Dirichlet L-functions of cube-free conductor`, Annals of Math, 192 (2) 2020. Statement taken from https://github. Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_erdos_supersingular_primes","problemName":"A conjecture of Erdős, supersingular primes and short character sums","description":"Unsolved. Group: formalization-evaluation. Source: M. Bennett and S. Siksek, `A conjecture of Erdős, supersingular primes and short character sums`, Annals of Math, 191 (2) 2020. Statement taken from https://git Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_mckay_conjecture","problemName":"The McKay Conjecture on character degrees","description":"Unsolved. Group: formalization-evaluation. Source: M. Cabanes and B. Späth, `The McKay Conjecture on character degrees`, Annals of Math, 203 (3) 2026. Statement taken from https://github.com/ImperialCollegeLondo Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_simplicity_conjecture","problemName":"Proof of the simplicity conjecture","description":"Unsolved. Group: formalization-evaluation. Source: D. Cristofaro-Gardiner, V. Humilière, and S. Seyfaddini, `Proof of the simplicity conjecture`, Annals of Math, 199 (1) 2024. Statement taken from https://github Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_spread_of_a_finite_group","problemName":"The spread of a finite group","description":"Unsolved. Group: formalization-evaluation. Source: T. C. Burness, R. M. Guralnick, and S. Harper, `The spread of a finite group`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/ImperialCol Problem by Katerina Hristova.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_symplectic_monodromy","problemName":"Symplectic monodromy at radius zero and equimultiplicity of μ-constant families","description":"Unsolved. Group: formalization-evaluation. Source: J. Fernández de Bobadilla and T. Pełka, `Symplectic monodromy at radius zero and equimultiplicity of μ-constant families`, Annals of Math, 200 (1) 2024. Stateme Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_uniform_mordell_lang","problemName":"Uniformity in Mordell–Lang for curves","description":"Unsolved. Group: formalization-evaluation. Source: V. Dimitrov, Z. Gao, and P. Habegger, `Uniformity in Mordell–Lang for curves`, Annals of Math, 194 (1) 2021. Statement taken from https://github.com/ImperialCol Problem by Thomas Browning, Christian Merten.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annulus_theorem_dim_four","problemName":"The Annulus Theorem in dimension 4 (Quinn)","description":"Unsolved. Group: formalization-evaluation. Source: F. Quinn, *Ends of maps III: dimensions 4 and 5*, J. Differential Geom. 17 (1982), building on M. H. Freedman, *The topology of four-dimensional manifolds*, J.  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annulus_theorem_high_dim","problemName":"The Annulus Theorem in dimension ≥ 5 (Kirby)","description":"Unsolved. Group: formalization-evaluation. Source: R. C. Kirby, *Stable homeomorphisms and the annulus conjecture*, Ann. of Math. 89 (1969). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-budney_gabai_knotted_three_spheres","problemName":"Budney--Gabai knotted three-spheres in S¹ × S³","description":"Unsolved. Group: formalization-evaluation. Source: R. Budney and D. Gabai, 'Knotted 3-balls in S⁴', Corollary 8.6, arXiv:1912.09029v3 (2021), https://arxiv.org/abs/1912.09029. Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-cerf_gamma_four","problemName":"Cerf's theorem: every self-diffeomorphism of S3 is smoothly isotopic to a linear isometry","description":"Unsolved. Group: formalization-evaluation. Source: J. Cerf, Sur les diffeomorphismes de la sphere de dimension trois, Lecture Notes in Mathematics 53, Springer (1968). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-classification_finite_simple_groups","problemName":"The classification of finite simple groups","description":"Unsolved. Group: formalization-evaluation. Source: D. Gorenstein, R. Lyons and R. Solomon, The Classification of the Finite Simple Groups (AMS Mathematical Surveys and Monographs 40, 1994–); M. Aschbacher, R. Ly Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-conway_knot_not_smoothly_slice","problemName":"The Conway knot is not smoothly slice","description":"Unsolved. Group: formalization-evaluation. Source: https://arxiv.org/abs/1808.02923 Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-conway_knot_topologically_slice","problemName":"The Conway knot is topologically slice","description":"Unsolved. Group: formalization-evaluation. Source: Freedman, *The topology of four-dimensional manifolds*, J. Diff. Geom. 17 (1982). See also Freedman-Quinn, *Topology of 4-Manifolds*, Princeton 1990. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-derived_solidification_free_CW_homology","problemName":"Derived solidification of free CW complexes (light condensed mathematics)","description":"Unsolved. Group: formalization-evaluation. Source: https://github.com/dagurtomas/LeanCondensed (LeanCondensed/Projects/DerivedSolidCWHomology.lean); D. Clausen and P. Scholze, lectures on analytic stacks and lig Problem by Dagur Asgeirsson.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-exists_topologically_slice_not_smoothly_slice","problemName":"Existence of a topologically slice, not smoothly slice knot","description":"Unsolved. Group: formalization-evaluation. Source: Casson, 1980s (unpublished); Akbulut-Matveyev, *A convex decomposition theorem for 4-manifolds*, IMRN 1998. See also Hedden-Kirk-Livingston, *Non-slice linear c Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-faltings","problemName":"Faltings' theorem (Mordell conjecture)","description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-fermat_last_theorem","problemName":"Fermat's Last Theorem","description":"Unsolved. Group: formalization-evaluation. Source: https://en.wikipedia.org/wiki/Fermat%27s_Last_Theorem Problem by Xuanji Li.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-five_transitive_card_classification","problemName":"Possible orders of 5-transitive finite permutation groups","description":"Unsolved. Group: formalization-evaluation. Source: Folklore via CFSG; classical work of Mathieu, Jordan; modern accounts in P. Cameron, Permutation Groups (1999). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-four_manifold_not_smooth","problemName":"Freedman's non-smoothability theorem","description":"Unsolved. Group: formalization-evaluation. Problem by Oliver Nash.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-kepler_conjecture","problemName":"Kepler conjecture (optimal sphere packing in ℝ³)","description":"Unsolved. Group: formalization-evaluation. Source: Kepler conjecture 1611 (J. Kepler, *Strena seu de Nive Sexangula*); proof T. Hales, 'A proof of the Kepler conjecture', Ann. of Math. (2) 162 (2005) 1065–1185;  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-kollar_lieblich_olsson_sawin","problemName":"Topological reconstruction theorems for varieties","description":"Unsolved. Group: formalization-evaluation. Source: János Kollár, Max Lieblich, Martin Olsson, and Will Sawin. The Zariski topology, linear systems, and algebraic varieties. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-manolescu_triangulation_disproof","problemName":"Manolescu's disproof of the triangulation conjecture","description":"Unsolved. Group: formalization-evaluation. Source: C. Manolescu, *Pin(2)-equivariant Seiberg–Witten Floer homology and the triangulation conjecture*, J. Amer. Math. Soc. 29 (2016), via the Galewski–Stern and Mat Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-mazur_torsion","problemName":"Mazur's torsion theorem","description":"Unsolved. Group: formalization-evaluation. Source: B. Mazur, *Modular curves and the Eisenstein ideal*, Publ. IHÉS 47 (1977). Knill, *Some fundamental theorems in mathematics*, §139. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-nikolov_segal","problemName":"Nikolov–Segal strong completeness theorem","description":"Unsolved. Group: formalization-evaluation. Source: Nikolay Nikolov and Dan Segal, *On finitely generated profinite groups, I: strong completeness and uniform bounds*, Annals of Mathematics 165 (2007), no. 1, 171 Problem by Adam Topaz.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-ore_conjecture","problemName":"The Ore conjecture: every element of a finite nonabelian simple group is a commutator","description":"Unsolved. Group: formalization-evaluation. Source: Martin W. Liebeck, E. A. O'Brien, Aner Shalev, and Pham Huu Tiep, *The Ore conjecture*, Journal of the European Mathematical Society 12 (2010), no. 4, 939–1008, Problem by Adam Topaz.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-poincare_3d_smooth","problemName":"3D smooth Poincaré conjecture (Perelman)","description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, three arXiv preprints 2002-2003 (math/0211159, math/0303109, math/0307245). Recorded as a `proof_wanted` in Mathlib/Geometry/Manifold/PoincareConje Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-poincare_3d_topological","problemName":"3D topological Poincaré conjecture (Perelman)","description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, The entropy formula for the Ricci flow and its geometric applications (arXiv:math/0211159, 2002); Ricci flow with surgery on three-manifolds (arXiv Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-poincare_4d_topological","problemName":"4D topological Poincaré conjecture (Freedman)","description":"Unsolved. Group: formalization-evaluation. Source: M. H. Freedman, The topology of four-dimensional manifolds, J. Differential Geom. 17 (1982), 357-453. Specialization of `proof_wanted ContinuousMap.HomotopyEqui Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-ramanujan_petersson","problemName":"Ramanujan–Petersson conjecture for the τ-function (Deligne's theorem)","description":"Unsolved. Group: formalization-evaluation. Source: S. Ramanujan, 'On certain arithmetical functions', Trans. Cambridge Philos. Soc. 22 (1916) 159–184. H. Petersson, 'Theorie der automorphen Formen beliebiger ree Problem by Seewoo Lee.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-schreier_conjecture","problemName":"Schreier's conjecture: outer automorphism group of a finite simple group is solvable","description":"Unsolved. Group: formalization-evaluation. Source: O. Schreier, Über die Erweiterung von Gruppen II, Abh. Math. Sem. Univ. Hamburg 4 (1926); CFSG, completed c. 2004. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-smale_conjecture","problemName":"Smale conjecture (Hatcher) in relative parameterized form","description":"Unsolved. Group: formalization-evaluation. Source: A. Hatcher, A proof of the Smale conjecture, Diff(S3) = O(4), Ann. of Math. 117 (1983). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-two_ninety_theorem","problemName":"The 290 theorem","description":"Unsolved. Group: formalization-evaluation. Source: M. Bhargava, J. Hanke, Universal quadratic forms and the 290-theorem, preprint (2011). See also https://en.wikipedia.org/wiki/15_and_290_theorems Problem by Bolton Bailey/Project Numina.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-watanabe_four_dim_smale_disproof","problemName":"Watanabe's disproof of the 4-dimensional Smale conjecture","description":"Unsolved. Group: formalization-evaluation. Source: T. Watanabe, *Some exotic nontrivial elements of the rational homotopy groups of Diff(S⁴)*, arXiv:1812.02448 (2018). Pairs with the positive S³ statement of Hat Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-weak_goldbach","problemName":"Weak Goldbach theorem","description":"Unsolved. Group: formalization-evaluation. Source: H. A. Helfgott, The ternary Goldbach conjecture is true, arXiv:1312.7748 (2013), https://arxiv.org/abs/1312.7748; H. A. Helfgott and D. J. Platt, Numerical Veri Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-weil_conjectures","problemName":"Weil conjectures in terms of point counts","description":"Unsolved. Group: formalization-evaluation. Source: P. Deligne, 'La conjecture de Weil. I', Publications Mathématiques de l'Institut des Hautes Études Scientifiques, Volume 43, pages 273–307 (1974), https://www.n Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-weinstein_conjecture_dim3","problemName":"Weinstein conjecture in dimension three (Taubes 2007)","description":"Unsolved. Group: formalization-evaluation. Source: C.H. Taubes, 'The Seiberg–Witten equations and the Weinstein conjecture', Geom. Topol. 11 (2007) 2117–2202; arXiv:math/0611007. Conjecture: A. Weinstein, 'On th Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"s2n-bignum-bench","competitionName":"s2n-bignum-bench","compute":"cpu","problemId":"tactic-synthesis","problemName":"HOL Light Tactic Synthesis","description":"Synthesize HOL Light tactic proofs for cryptographic theorems from AWS s2n-bignum.","metricName":"accuracy","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench Leaderboard","compute":"unknown","problemId":"overall-accuracy","problemName":"Overall Accuracy Track","metricName":"Overall Accuracy","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench Leaderboard","compute":"unknown","problemId":"multilabel-f1","problemName":"Multilabel F1 Score Track","metricName":"Multilabel F1 Score","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench Leaderboard","compute":"unknown","problemId":"multiclass-jaccard","problemName":"Multiclass Jaccard Index Track","metricName":"Multiclass Jaccard Index","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench Leaderboard","compute":"unknown","problemId":"biomassters-rmse","problemName":"Biomassters RMSE Track","metricName":"RMSE","metricDirection":"minimize"},{"competitionId":"apex-humanoid-parkour","competitionName":"Humanoid Parkour","compute":"cpu","problemId":"humanoid-parkour-course","problemName":"Humanoid Parkour Course","description":"Drive a Unitree G1 humanoid through a 51 m parkour course with randomized friction and wind.","metricName":"score","metricDirection":"maximize","baseline":0.21},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"causal-conv1d","problemName":"Causal Conv1d","description":"Optimize the 1D causal convolution kernel, commonly used in modern sequence models like Mamba.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"gated-deltanet-chunk-fwd-o","problemName":"Gated Deltanet chunk_fwd_o","description":"Optimize the forward pass chunk output kernel for Gated Deltanet.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"qr-v2","problemName":"QR Decomposition v2","description":"Optimize QR decomposition with a dedicated Nsight Compute-compatible profile path.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"cholesky","problemName":"Cholesky Validation","description":"Optimize Cholesky decomposition with application validation over natural-gradient logistic-regression loops.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"eigh","problemName":"Eigh Validation","description":"Optimize symmetric eigenvalue decomposition (eigh) validation kernels.","metricName":"time","metricDirection":"minimize"},{"competitionId":"medmcqa","competitionName":"MedMCQA","compute":"unknown","problemId":"medmcqa-test-accuracy","problemName":"MedMCQA Test Accuracy","description":"Accuracy on the private test set consisting of AIIMS PG exam MCQs.","metricName":"accuracy","metricDirection":"maximize"}]}