{ "generated_at": "2026-11-03T16:38:16.780832+00:00", "count": 281, "categories": [ { "name": "Observability", "count": 61 }, { "name": "Red Teaming & Safety", "count": 59 }, { "name": "RAG Eval", "count": 43 }, { "name": "Agent Eval", "count": 42 }, { "name": "Benchmarks", "count": 35 }, { "name": "Eval Frameworks", "count": 30 }, { "name": "Coding Eval", "count": 8 }, { "name": "Reasoning", "count": 3 } ], "items": [ { "name": "iFixAi", "full_name": "ifixai-ai/iFixAi", "slug": "ifixai-ai-ifixai", "url": "https://github.com/ifixai-ai/iFixAi", "owner": "ifixai-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/279006089?v=4", "stars": 43076, "forks": 1781, "open_issues": 6, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T17:18:51Z", "created_at": "2026-04-27T13:02:34Z", "description": "Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.", "topics": [ "agent-evaluation", "ai", "ai-alignment", "ai-evaluation", "ai-governance", "ai-safety", "cli", "diagnostic-tool", "eu-ai-act", "hallucination-detection", "iso-42001", "llm-evaluation", "llm-security", "nist-ai-rmf", "owasp-llm", "prompt-injection", "python", "responsible-ai", "risk-assessment", "risk-management" ], "category": "Red Teaming & Safety", "momentum": 87, "rank": 1 }, { "name": "langfuse", "full_name": "langfuse/langfuse", "slug": "langfuse-langfuse", "url": "https://github.com/langfuse/langfuse", "owner": "langfuse", "owner_avatar": "https://avatars.githubusercontent.com/u/134601687?v=4", "stars": 36955, "forks": 4149, "open_issues": 1094, "language": "TypeScript", "license": "NOASSERTION", "pushed_at": "2026-11-03T16:30:09Z", "created_at": "2023-05-18T17:47:09Z", "description": "🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.", "topics": [ "analytics", "autogen", "evaluation", "langchain", "large-language-models", "llama-index", "llm", "llm-evaluation", "llm-observability", "llmops", "monitoring", "observability", "open-source", "openai", "playground", "prompt-engineering", "prompt-management", "self-hosted", "ycombinator" ], "category": "Observability", "momentum": 86, "rank": 2 }, { "name": "WeKnora", "full_name": "Tencent/WeKnora", "slug": "tencent-weknora", "url": "https://github.com/Tencent/WeKnora", "owner": "Tencent", "owner_avatar": "https://avatars.githubusercontent.com/u/18461506?v=4", "stars": 36379, "forks": 4964, "open_issues": 770, "language": "Go", "license": "NOASSERTION", "pushed_at": "2026-11-03T03:33:48Z", "created_at": "2025-07-22T08:01:23Z", "description": "Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.", "topics": [ "agent", "agentic", "ai", "chatbot", "dsh-plugin", "embeddings", "evaluation", "generative-ai", "golang", "knowledge-base", "llm", "multi-tenant", "ollama", "openai", "question-answering", "rag", "reranking", "semantic-search", "vector-search", "wiki" ], "category": "RAG Eval", "momentum": 86, "rank": 3 }, { "name": "mlflow", "full_name": "mlflow/mlflow", "slug": "mlflow-mlflow", "url": "https://github.com/mlflow/mlflow", "owner": "mlflow", "owner_avatar": "https://avatars.githubusercontent.com/u/39938107?v=4", "stars": 28647, "forks": 6576, "open_issues": 2226, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:34:06Z", "created_at": "2018-06-05T16:05:58Z", "description": "The open source AI engineering platform for agents, LLMs, and ML models. MLflow enables teams of all sizes to debug, evaluate, monitor, and optimize production-quality AI applications while controlling costs and managing access to models and data.", "topics": [ "agentops", "agents", "ai", "ai-governance", "apache-spark", "evaluation", "langchain", "llm-evaluation", "llmops", "machine-learning", "ml", "mlflow", "mlops", "model-management", "observability", "open-source", "openai", "prompt-engineering" ], "category": "Observability", "momentum": 85, "rank": 4 }, { "name": "promptfoo", "full_name": "promptfoo/promptfoo", "slug": "promptfoo-promptfoo", "url": "https://github.com/promptfoo/promptfoo", "owner": "promptfoo", "owner_avatar": "https://avatars.githubusercontent.com/u/137907881?v=4", "stars": 26791, "forks": 2642, "open_issues": 908, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-11-03T08:29:35Z", "created_at": "2023-04-28T15:48:49Z", "description": "Test your prompts, agents, and RAGs. Red teaming/pentesting/vulnerability scanning for AI. Compare performance of GPT, Claude, Gemini, DeepSeek, and more. Simple declarative configs with command line and CI/CD integration. Used by OpenAI and Anthropic.", "topics": [ "ci", "ci-cd", "cicd", "evaluation", "evaluation-framework", "llm", "llm-eval", "llm-evaluation", "llm-evaluation-framework", "llmops", "pentesting", "prompt-engineering", "prompt-testing", "prompts", "rag", "red-teaming", "testing", "vulnerability-scanners" ], "category": "Red Teaming & Safety", "momentum": 84, "rank": 5 }, { "name": "opik", "full_name": "comet-ml/opik", "slug": "comet-ml-opik", "url": "https://github.com/comet-ml/opik", "owner": "comet-ml", "owner_avatar": "https://avatars.githubusercontent.com/u/31487821?v=4", "stars": 23146, "forks": 1925, "open_issues": 194, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:35:04Z", "created_at": "2023-05-10T12:57:13Z", "description": "Debug, evaluate, and monitor your LLM applications, RAG systems, and agentic workflows with comprehensive tracing, automated evaluations, and production-ready dashboards.", "topics": [ "evaluation", "hacktoberfest", "hacktoberfest2025", "langchain", "llama-index", "llm", "llm-evaluation", "llm-observability", "llmops", "open-source", "openai", "playground", "prompt-engineering" ], "category": "Observability", "momentum": 84, "rank": 6 }, { "name": "openobserve", "full_name": "openobserve/openobserve", "slug": "openobserve-openobserve", "url": "https://github.com/openobserve/openobserve", "owner": "openobserve", "owner_avatar": "https://avatars.githubusercontent.com/u/95867656?v=4", "stars": 22762, "forks": 1179, "open_issues": 225, "language": "TypeScript", "license": "AGPL-3.0", "pushed_at": "2026-11-03T16:37:53Z", "created_at": "2023-02-02T03:29:51Z", "description": "Open source observability platform for logs, metrics, traces, RUM (web, android, ios), Session replay, pipelines, SLO and LLM observability. A sophisticated, simple and highly performant alternative to Datadog, Splunk, and Elasticsearch with 140x lower storage costs and single binary deployment.", "topics": [ "analytics", "apm", "datadog", "elasticsearch", "grafana", "jaeger", "kibana", "log-analytics", "log-management", "log-search", "logs", "metrics", "monitoring", "observability", "openobserve", "opentelemetry", "prometheus", "rust-lang", "splunk", "traces" ], "category": "Observability", "momentum": 84, "rank": 7 }, { "name": "deepeval", "full_name": "confident-ai/deepeval", "slug": "confident-ai-deepeval", "url": "https://github.com/confident-ai/deepeval", "owner": "confident-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/130858411?v=4", "stars": 19272, "forks": 2164, "open_issues": 802, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T19:23:56Z", "created_at": "2023-08-10T05:35:04Z", "description": "The LLM Evaluation Framework", "topics": [ "evaluation-framework", "evaluation-metrics", "llm-evaluation", "llm-evaluation-framework", "llm-evaluation-metrics", "python" ], "category": "Eval Frameworks", "momentum": 83, "rank": 8 }, { "name": "phoenix", "full_name": "Arize-ai/phoenix", "slug": "arize-ai-phoenix", "url": "https://github.com/Arize-ai/phoenix", "owner": "Arize-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/59858760?v=4", "stars": 12120, "forks": 1260, "open_issues": 1215, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-03T16:37:23Z", "created_at": "2022-11-09T23:44:35Z", "description": "AI Observability & Evaluation", "topics": [ "agents", "ai-monitoring", "ai-observability", "aiengineering", "anthropic", "datasets", "evals", "langchain", "llamaindex", "llm-eval", "llm-evaluation", "llmops", "llms", "openai", "prompt-engineering", "smolagents" ], "category": "Observability", "momentum": 80, "rank": 9 }, { "name": "bisheng", "full_name": "dataelement/bisheng", "slug": "dataelement-bisheng", "url": "https://github.com/dataelement/bisheng", "owner": "dataelement", "owner_avatar": "https://avatars.githubusercontent.com/u/103249391?v=4", "stars": 12110, "forks": 1993, "open_issues": 16, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:19:38Z", "created_at": "2023-08-28T10:00:24Z", "description": "BISHENG is an open LLM devops platform for next generation Enterprise AI applications. Powerful and comprehensive features include: GenAI workflow, RAG, Agent, Unified model management, Evaluation, SFT, Dataset Management, Enterprise-level System Management, Observability and more.", "topics": [ "agent", "ai", "chatbot", "enterprise", "finetune", "genai", "gpt", "langchian", "llama", "llm", "llmdevops", "llmops", "ocr", "openai", "orchestration", "python", "rag", "react", "sft", "workflow" ], "category": "Observability", "momentum": 80, "rank": 10 }, { "name": "garak", "full_name": "NVIDIA/garak", "slug": "nvidia-garak", "url": "https://github.com/NVIDIA/garak", "owner": "NVIDIA", "owner_avatar": "https://avatars.githubusercontent.com/u/1728152?v=4", "stars": 9846, "forks": 1439, "open_issues": 542, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T19:33:43Z", "created_at": "2023-05-10T18:52:16Z", "description": "the LLM vulnerability scanner", "topics": [ "ai", "llm-evaluation", "llm-security", "security-scanners", "vulnerability-assessment" ], "category": "Eval Frameworks", "momentum": 79, "rank": 11 }, { "name": "oumi", "full_name": "oumi-ai/oumi", "slug": "oumi-ai-oumi", "url": "https://github.com/oumi-ai/oumi", "owner": "oumi-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/167452922?v=4", "stars": 9421, "forks": 798, "open_issues": 28, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T00:42:25Z", "created_at": "2024-05-07T17:45:15Z", "description": "Easily fine-tune (SFT/RL), evaluate, and deploy Qwen, Gemma, or any open agentic LLM/VLM!", "topics": [ "agentic-ai", "dpo", "fine-tuning", "gemma", "gpt-oss", "grpo", "inference", "llama", "llm", "llm-as-a-judge", "llm-evaluation", "llms", "lora", "pytorch", "qwen", "reinforcement-learning", "sft", "synthetic-data", "tool-calling", "vlm" ], "category": "Agent Eval", "momentum": 79, "rank": 12 }, { "name": "awesome-prompts", "full_name": "ai-boost/awesome-prompts", "slug": "ai-boost-awesome-prompts", "url": "https://github.com/ai-boost/awesome-prompts", "owner": "ai-boost", "owner_avatar": "https://avatars.githubusercontent.com/u/70333750?v=4", "stars": 9174, "forks": 895, "open_issues": 0, "language": "Python", "license": "GPL-3.0", "pushed_at": "2026-11-03T08:39:27Z", "created_at": "2024-01-19T15:15:10Z", "description": "Curated prompts, frameworks and papers for LLMs and agents, with an engineering bias. Prompt engineering, context engineering, agent harnesses, red teaming.", "topics": [ "agents", "awesome", "awesome-list", "chatgpt", "context-engineering", "llm", "papers", "prompt", "prompt-engineering" ], "category": "Red Teaming & Safety", "momentum": 79, "rank": 13 }, { "name": "Tracely-ai", "full_name": "jwuthri-cloud/Tracely-ai", "slug": "jwuthri-cloud-tracely-ai", "url": "https://github.com/jwuthri-cloud/Tracely-ai", "owner": "jwuthri-cloud", "owner_avatar": "https://avatars.githubusercontent.com/u/341043007?v=4", "stars": 1213, "forks": 141, "open_issues": 21, "language": "Python", "license": "MIT", "pushed_at": "2026-11-02T18:00:10Z", "created_at": "2026-10-11T15:38:03Z", "description": "Trace-native CI/CD for AI agents: grade production traces, cluster failures, freeze them into replayable regression tests, and block PRs that regress. Self-hostable.", "topics": [ "agent-evaluation", "ai-agents", "ai-testing", "ci-cd", "clickhouse", "developer-tools", "llm-evaluation", "llm-observability", "llmops", "mcp", "observability", "python", "regression-testing", "self-hosted", "tracing" ], "category": "Observability", "momentum": 79, "rank": 14 }, { "name": "opencompass", "full_name": "open-compass/opencompass", "slug": "open-compass-opencompass", "url": "https://github.com/open-compass/opencompass", "owner": "open-compass", "owner_avatar": "https://avatars.githubusercontent.com/u/143521324?v=4", "stars": 7563, "forks": 900, "open_issues": 337, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T02:27:10Z", "created_at": "2023-06-15T12:42:58Z", "description": "OpenCompass is an LLM evaluation platform, supporting a wide range of models from OpenAI, Anthropic, Gemini, Qwen, GLM, DeepSeek, etc, across 100+ datasets covering knowledge, reasoning, coding, science, language, long-context, and safety.", "topics": [ "benchmark", "evaluation", "large-language-model", "llm", "llm-eval" ], "category": "Red Teaming & Safety", "momentum": 78, "rank": 15 }, { "name": "openllmetry", "full_name": "traceloop/openllmetry", "slug": "traceloop-openllmetry", "url": "https://github.com/traceloop/openllmetry", "owner": "traceloop", "owner_avatar": "https://avatars.githubusercontent.com/u/125419530?v=4", "stars": 7519, "forks": 1185, "open_issues": 832, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T13:34:23Z", "created_at": "2023-09-02T14:42:59Z", "description": "Open-source observability for your GenAI or LLM application, based on OpenTelemetry", "topics": [ "artifical-intelligence", "datascience", "generative-ai", "good-first-issue", "good-first-issues", "help-wanted", "llm", "llmops", "metrics", "ml", "model-monitoring", "monitoring", "observability", "open-source", "open-telemetry", "opentelemetry", "opentelemetry-python", "python" ], "category": "Observability", "momentum": 78, "rank": 16 }, { "name": "AI-Infra-Guard", "full_name": "Tencent/AI-Infra-Guard", "slug": "tencent-ai-infra-guard", "url": "https://github.com/Tencent/AI-Infra-Guard", "owner": "Tencent", "owner_avatar": "https://avatars.githubusercontent.com/u/18461506?v=4", "stars": 7366, "forks": 730, "open_issues": 24, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T02:07:41Z", "created_at": "2024-12-25T06:39:30Z", "description": "A full-stack AI Red Teaming platform securing AI ecosystems via Agent Scan, Skills Scan, MCP scan, AI Infra scan and LLM jailbreak evaluation.", "topics": [ "agent", "agent-security", "agentic-ai", "ai-infra", "ai-red-teaming", "ai-safety", "ai-security", "llm", "llm-evaluation", "llm-jailbreak", "llm-security", "mcp-scan", "openclaw-security", "prompt-injection", "prompt-security", "scanner", "security", "security-tools", "skill-scanner", "vulnerability" ], "category": "Red Teaming & Safety", "momentum": 78, "rank": 17 }, { "name": "whichllm", "full_name": "Andyyyy64/whichllm", "slug": "andyyyy64-whichllm", "url": "https://github.com/Andyyyy64/whichllm", "owner": "Andyyyy64", "owner_avatar": "https://avatars.githubusercontent.com/u/105579829?v=4", "stars": 6818, "forks": 376, "open_issues": 5, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T08:51:13Z", "created_at": "2026-03-04T13:16:00Z", "description": "Find the local LLM that actually runs and performs best on your hardware. Ranked by real, recency-aware benchmarks, not parameter count. One command, run it instantly.", "topics": [ "localllm" ], "category": "Benchmarks", "momentum": 77, "rank": 18 }, { "name": "harbor", "full_name": "harbor-framework/harbor", "slug": "harbor-framework-harbor", "url": "https://github.com/harbor-framework/harbor", "owner": "harbor-framework", "owner_avatar": "https://avatars.githubusercontent.com/u/256467836?v=4", "stars": 6755, "forks": 2133, "open_issues": 1209, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T08:54:25Z", "created_at": "2025-08-04T23:28:26Z", "description": "Framework for evaluating and improving agents", "topics": [ "evals", "rl-environments", "terminal-bench" ], "category": "Agent Eval", "momentum": 77, "rank": 19 }, { "name": "ouroboros", "full_name": "Q00/ouroboros", "slug": "q00-ouroboros", "url": "https://github.com/Q00/ouroboros", "owner": "Q00", "owner_avatar": "https://avatars.githubusercontent.com/u/31264094?v=4", "stars": 6309, "forks": 640, "open_issues": 79, "language": "Python", "license": "MIT", "pushed_at": "2026-11-01T19:58:39Z", "created_at": "2026-01-14T01:14:04Z", "description": "Agent OS: the agent gets smarter on its own. We just hold the line: Interview-gated, staged evaluation, budgeted evolution loop. MCP server, 14 runtimes: Claude Code, Codex CLI, Gemini CLI, OpenCode, Copilot, Kiro and more.", "topics": [ "agent-os", "agentic-ai", "ai-agent", "ai-coding-agent", "claude-code", "cli", "codex", "coding-agent", "deepseek", "developer-tools", "dsh", "dsh-plugin", "llm-evaluation", "llm-orchestration", "loop-engineering", "mcp", "opencode", "skill", "skills" ], "category": "Agent Eval", "momentum": 77, "rank": 20 }, { "name": "giskard-oss", "full_name": "Giskard-AI/giskard-oss", "slug": "giskard-ai-giskard-oss", "url": "https://github.com/Giskard-AI/giskard-oss", "owner": "Giskard-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/71782571?v=4", "stars": 5942, "forks": 560, "open_issues": 78, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T07:34:45Z", "created_at": "2022-03-06T21:45:37Z", "description": "🐢 Open-Source Evaluation & Testing library for LLM Agents", "topics": [ "agent-evaluation", "ai-red-team", "ai-security", "ai-testing", "fairness-ai", "llm", "llm-eval", "llm-evaluation", "llm-security", "llmops", "ml-testing", "ml-validation", "mlops", "rag-evaluation", "red-team-tools", "responsible-ai", "trustworthy-ai" ], "category": "Red Teaming & Safety", "momentum": 77, "rank": 21 }, { "name": "coze-loop", "full_name": "coze-dev/coze-loop", "slug": "coze-dev-coze-loop", "url": "https://github.com/coze-dev/coze-loop", "owner": "coze-dev", "owner_avatar": "https://avatars.githubusercontent.com/u/157483752?v=4", "stars": 5800, "forks": 803, "open_issues": 100, "language": "Go", "license": "Apache-2.0", "pushed_at": "2026-11-03T15:48:53Z", "created_at": "2025-06-24T00:26:28Z", "description": "Next-generation AI Agent Optimization Platform: Cozeloop addresses challenges in AI agent development by providing full-lifecycle management capabilities from development, debugging, and evaluation to monitoring.", "topics": [ "agent", "agent-evaluation", "agent-observability", "agentops", "ai", "coze", "eino", "evaluation", "langchain", "llm-observability", "llmops", "monitoring", "observability", "open-source", "openai", "playground", "prompt-management" ], "category": "Observability", "momentum": 77, "rank": 22 }, { "name": "plano", "full_name": "katanemo/plano", "slug": "katanemo-plano", "url": "https://github.com/katanemo/plano", "owner": "katanemo", "owner_avatar": "https://avatars.githubusercontent.com/u/112724757?v=4", "stars": 7107, "forks": 494, "open_issues": 155, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-10-24T19:03:25Z", "created_at": "2024-07-09T20:25:56Z", "description": "Plano is an AI-native proxy server and data plane for agentic apps. Smart LLM routing, observability, agent orchestration, and guardrails so you stay focused on your agents core logic.", "topics": [ "ai-gateway", "ai-gateway-support", "envoy", "envoyproxy", "gateway", "generative-ai", "llm-gateway", "llm-inference", "llm-proxy", "llm-routing", "llmops", "llms", "openai", "prompt", "proxy", "proxy-server", "routing" ], "category": "Observability", "momentum": 76, "rank": 23 }, { "name": "chinese-llm-benchmark", "full_name": "jeinlee1991/chinese-llm-benchmark", "slug": "jeinlee1991-chinese-llm-benchmark", "url": "https://github.com/jeinlee1991/chinese-llm-benchmark", "owner": "jeinlee1991", "owner_avatar": "https://avatars.githubusercontent.com/u/46815718?v=4", "stars": 6501, "forks": 262, "open_issues": 19, "language": "", "license": "", "pushed_at": "2026-10-27T04:49:07Z", "created_at": "2023-06-04T07:23:20Z", "description": "非线智能 NoneLinear - ReLE评测：中文AI大模型能力评测（持续更新）：目前已囊括374个大模型，覆盖chatgpt、gpt-5.4、谷歌gemini-3.1-pro、Claude-4.6、文心ERNIE-X1.1、ERNIE-5.0、qwen3.6-max、qwen3.6-plus、百川、讯飞星火、商汤senseChat等商用模型， 以及step3.5-flash、kimi-k2.6、ernie4.5、MiniMax-M2.7、deepseek-v4、Qwen3.6、llama4、智谱GLM-5.1、MiMo-V2、LongCat、gemma4、mistral等开源大模型。不仅提供排行榜，也提供规模超200万的大模型缺陷库！方便广大社区研究分析、改进大模型。", "topics": [ "agentic-ai", "artificial-intelligence", "llm-agent", "llm-evaluation" ], "category": "Agent Eval", "momentum": 76, "rank": 24 }, { "name": "helicone", "full_name": "Helicone/helicone", "slug": "helicone-helicone", "url": "https://github.com/Helicone/helicone", "owner": "Helicone", "owner_avatar": "https://avatars.githubusercontent.com/u/114524975?v=4", "stars": 6257, "forks": 702, "open_issues": 188, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2026-10-28T15:59:52Z", "created_at": "2023-01-31T22:34:44Z", "description": "🧊 Open source LLM observability platform. One line of code to monitor, evaluate, and experiment. YC W23 🍓", "topics": [ "agent-monitoring", "analytics", "evaluation", "gpt", "langchain", "large-language-models", "llama-index", "llm", "llm-cost", "llm-evaluation", "llm-observability", "llmops", "monitoring", "open-source", "openai", "playground", "prompt-engineering", "prompt-management", "ycombinator" ], "category": "Observability", "momentum": 76, "rank": 25 }, { "name": "Kiln", "full_name": "Kiln-AI/Kiln", "slug": "kiln-ai-kiln", "url": "https://github.com/Kiln-AI/Kiln", "owner": "Kiln-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/178670964?v=4", "stars": 5253, "forks": 398, "open_issues": 41, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-03T16:18:22Z", "created_at": "2024-07-23T23:10:13Z", "description": "Build, Evaluate, and Optimize AI Systems. Includes evals, RAG, agents, fine-tuning, synthetic data generation, dataset management, MCP, and more.", "topics": [ "ai", "chain-of-thought", "collaboration", "dataset-generation", "evals", "evaluation", "evaluation-framework", "fine-tuning", "machine-learning", "macos", "mcp", "ml", "ollama", "openai", "prompt", "prompt-engineering", "python", "rlhf", "synthetic-data", "windows" ], "category": "RAG Eval", "momentum": 76, "rank": 26 }, { "name": "langwatch", "full_name": "langwatch/langwatch", "slug": "langwatch-langwatch", "url": "https://github.com/langwatch/langwatch", "owner": "langwatch", "owner_avatar": "https://avatars.githubusercontent.com/u/146763322?v=4", "stars": 5009, "forks": 426, "open_issues": 424, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:28:11Z", "created_at": "2023-09-09T11:33:18Z", "description": "Open-Source Agent Observability & Evals for AI Platform teams. Trace, test, route and govern every agent & coding assistant in the company.", "topics": [ "agent-testing", "ai", "analytics", "datasets", "dspy", "evaluation", "gpt", "llm", "llm-ops", "llmops", "low-code", "observability", "observability-platform", "openai", "prompt-engineering", "simulation-testing" ], "category": "Observability", "momentum": 76, "rank": 27 }, { "name": "latitude-llm", "full_name": "latitude-dev/latitude-llm", "slug": "latitude-dev-latitude-llm", "url": "https://github.com/latitude-dev/latitude-llm", "owner": "latitude-dev", "owner_avatar": "https://avatars.githubusercontent.com/u/98949449?v=4", "stars": 4744, "forks": 408, "open_issues": 114, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-11-03T16:08:20Z", "created_at": "2024-06-28T11:16:02Z", "description": "Open-source observability for AI agents. Find where your agents fail, dispatch your coding agent to fix it, and verify the fix against real traces.", "topics": [ "agent-monitoring", "agent-observability", "ai-error-monitor", "ai-monitoring", "ai-observability", "llm-error-tracing", "llm-observability", "llm-tools" ], "category": "Observability", "momentum": 76, "rank": 28 }, { "name": "logfire", "full_name": "pydantic/logfire", "slug": "pydantic-logfire", "url": "https://github.com/pydantic/logfire", "owner": "pydantic", "owner_avatar": "https://avatars.githubusercontent.com/u/110818415?v=4", "stars": 4532, "forks": 351, "open_issues": 236, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T14:26:41Z", "created_at": "2024-04-23T11:50:23Z", "description": "AI observability platform for production LLM and agent systems.", "topics": [ "agent-observability", "ai", "ai-observability", "ai-tools", "evals", "fastapi", "llm-observability", "logging", "metrics", "observability", "openai", "opentelemetry", "pydantic", "pydantic-ai", "python", "trace" ], "category": "Observability", "momentum": 75, "rank": 29 }, { "name": "Spiritual-Spell-Red-Teaming", "full_name": "Goochbeater/Spiritual-Spell-Red-Teaming", "slug": "goochbeater-spiritual-spell-red-teaming", "url": "https://github.com/Goochbeater/Spiritual-Spell-Red-Teaming", "owner": "Goochbeater", "owner_avatar": "https://avatars.githubusercontent.com/u/174131698?v=4", "stars": 4494, "forks": 795, "open_issues": 27, "language": "", "license": "", "pushed_at": "2026-11-02T01:10:59Z", "created_at": "2025-12-15T22:17:23Z", "description": "A repo for jailbreaking various LLMs, mainly Claude", "topics": [], "category": "Red Teaming & Safety", "momentum": 75, "rank": 30 }, { "name": "lmms-eval", "full_name": "EvolvingLMMs-Lab/lmms-eval", "slug": "evolvinglmms-lab-lmms-eval", "url": "https://github.com/EvolvingLMMs-Lab/lmms-eval", "owner": "EvolvingLMMs-Lab", "owner_avatar": "https://avatars.githubusercontent.com/u/154951679?v=4", "stars": 4480, "forks": 683, "open_issues": 33, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-03T14:44:30Z", "created_at": "2024-03-07T12:09:25Z", "description": "One-for-All Multimodal Evaluation Toolkit Across Text, Image, Video, and Audio Tasks", "topics": [ "agi", "audio-evaluation", "benchmark", "evaluation", "large-language-models", "llm-evaluation", "multimodal", "multimodal-evaluation", "video-understanding", "vision-language-model", "vlm" ], "category": "Benchmarks", "momentum": 75, "rank": 31 }, { "name": "VLMEvalKit", "full_name": "open-compass/VLMEvalKit", "slug": "open-compass-vlmevalkit", "url": "https://github.com/open-compass/VLMEvalKit", "owner": "open-compass", "owner_avatar": "https://avatars.githubusercontent.com/u/143521324?v=4", "stars": 4472, "forks": 788, "open_issues": 325, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T10:30:26Z", "created_at": "2023-12-01T08:18:11Z", "description": "Open-source evaluation toolkit of large multi-modality models (LMMs), support 220+ LMMs, 80+ benchmarks", "topics": [ "chatgpt", "claude", "clip", "computer-vision", "evaluation", "gemini", "gpt", "gpt-4v", "gpt4", "large-language-models", "llava", "llm", "multi-modal", "openai", "openai-api", "pytorch", "qwen", "vit", "vqa" ], "category": "Benchmarks", "momentum": 75, "rank": 32 }, { "name": "solace-agent-mesh", "full_name": "SolaceProducts/solace-agent-mesh", "slug": "solaceproducts-solace-agent-mesh", "url": "https://github.com/SolaceProducts/solace-agent-mesh", "owner": "SolaceProducts", "owner_avatar": "https://avatars.githubusercontent.com/u/32577848?v=4", "stars": 4921, "forks": 272, "open_issues": 0, "language": "Shell", "license": "NOASSERTION", "pushed_at": "2026-10-23T15:09:08Z", "created_at": "2025-01-10T17:11:47Z", "description": "Platform for building and running AI agents across the full lifecycle — build, test, deploy, monitor, improve. Hosts your agents, runs their tools, and coordinates them over an event mesh, with sessions, artifacts, auth, evaluators, and observability built in.", "topics": [ "a2a", "a2a-protocol", "agent-mesh", "agentic-ai", "agents", "ai-agents", "event-driven", "framework", "go", "harness", "mcp", "multi-agent", "multi-agent-systems", "python", "solace", "workflows" ], "category": "Observability", "momentum": 74, "rank": 33 }, { "name": "evalscope", "full_name": "modelscope/evalscope", "slug": "modelscope-evalscope", "url": "https://github.com/modelscope/evalscope", "owner": "modelscope", "owner_avatar": "https://avatars.githubusercontent.com/u/109945100?v=4", "stars": 3641, "forks": 552, "open_issues": 17, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T15:16:42Z", "created_at": "2023-12-07T06:10:49Z", "description": "A streamlined and customizable framework for efficient large model (LLM, VLM, AIGC) evaluation and performance benchmarking.", "topics": [ "evaluation", "llm", "performance", "rag", "vlm" ], "category": "RAG Eval", "momentum": 74, "rank": 34 }, { "name": "trulens", "full_name": "truera/trulens", "slug": "truera-trulens", "url": "https://github.com/truera/trulens", "owner": "truera", "owner_avatar": "https://avatars.githubusercontent.com/u/51224128?v=4", "stars": 3628, "forks": 385, "open_issues": 137, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T02:43:30Z", "created_at": "2020-11-02T21:56:45Z", "description": "Evaluation and Tracking for LLM Experiments and AI Agents", "topics": [ "agent-evaluation", "agentops", "ai-agents", "ai-monitoring", "ai-observability", "evals", "explainable-ml", "llm-eval", "llm-evaluation", "llmops", "llms", "machine-learning", "neural-networks" ], "category": "Observability", "momentum": 74, "rank": 35 }, { "name": "Switchyard", "full_name": "NVIDIA-NeMo/Switchyard", "slug": "nvidia-nemo-switchyard", "url": "https://github.com/NVIDIA-NeMo/Switchyard", "owner": "NVIDIA-NeMo", "owner_avatar": "https://avatars.githubusercontent.com/u/213689629?v=4", "stars": 3479, "forks": 351, "open_issues": 102, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-11-03T06:42:33Z", "created_at": "2026-05-19T19:06:33Z", "description": "Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.", "topics": [], "category": "Benchmarks", "momentum": 74, "rank": 36 }, { "name": "lmnr", "full_name": "lmnr-ai/lmnr", "slug": "lmnr-ai-lmnr", "url": "https://github.com/lmnr-ai/lmnr", "owner": "lmnr-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/161496104?v=4", "stars": 3472, "forks": 258, "open_issues": 154, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:20:56Z", "created_at": "2024-08-29T03:45:28Z", "description": "Laminar - open-source observability platform purpose-built for AI agents. YC S24.", "topics": [ "agent-observability", "agents", "ai", "ai-observability", "aiops", "analytics", "developer-tools", "evals", "evaluation", "llm-evaluation", "llm-observability", "llmops", "monitoring", "observability", "open-source", "rust", "rust-lang", "self-hosted", "ts", "typescript" ], "category": "Observability", "momentum": 74, "rank": 37 }, { "name": "factcircuit", "full_name": "superwesleyhys-ux/factcircuit", "slug": "superwesleyhys-ux-factcircuit", "url": "https://github.com/superwesleyhys-ux/factcircuit", "owner": "superwesleyhys-ux", "owner_avatar": "https://avatars.githubusercontent.com/u/251160695?v=4", "stars": 932, "forks": 73, "open_issues": 7, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T07:05:55Z", "created_at": "2026-09-06T09:15:33Z", "description": "FactCircuit: an auditable claim-and-evidence verification loop with immutable provenance, staged checks, bounded retrieval, and reproducible evaluation.", "topics": [ "accuracy-decline", "ai-safety", "audit-trail", "fact-checking", "factcircuit", "information-retrieval", "llm-evaluation", "news-verification", "provenance", "python", "reproducible-research" ], "category": "Red Teaming & Safety", "momentum": 74, "rank": 38 }, { "name": "awesome-jev-tools", "full_name": "v-modal/awesome-jev-tools", "slug": "v-modal-awesome-jev-tools", "url": "https://github.com/v-modal/awesome-jev-tools", "owner": "v-modal", "owner_avatar": "https://avatars.githubusercontent.com/u/286898453?v=4", "stars": 753, "forks": 56, "open_issues": 3, "language": "", "license": "", "pushed_at": "2026-11-03T15:09:45Z", "created_at": "2026-09-19T16:48:59Z", "description": "A curated list of tools built for Jev — TypeSafe AI's System One model for typed decisions.", "topics": [ "awesome-ai", "awesome-jev", "awesome-list", "awesome-readme", "calibrated-probabilities", "jev", "jev-ai", "jev-alternative", "jev-api", "jev-connectors", "jev-model", "jev-production", "jev-test", "jev-tools", "jev-typesafe", "laya", "llm-evaluation", "llm-inference", "llm-tools", "vmodal" ], "category": "Eval Frameworks", "momentum": 74, "rank": 39 }, { "name": "ai-engineering-course", "full_name": "amitshekhariitbhu/ai-engineering-course", "slug": "amitshekhariitbhu-ai-engineering-course", "url": "https://github.com/amitshekhariitbhu/ai-engineering-course", "owner": "amitshekhariitbhu", "owner_avatar": "https://avatars.githubusercontent.com/u/9877145?v=4", "stars": 628, "forks": 123, "open_issues": 0, "language": "Markdown", "license": "Apache-2.0", "pushed_at": "2026-11-03T06:23:53Z", "created_at": "2026-09-23T08:04:42Z", "description": "AI Engineering Course - A free and complete AI Engineering Course to learn AI Engineering step by step - from Machine Learning, Neural Networks, and Transformers to LLMs, Fine-Tuning, RAG, AI Agents, LLM Inference, Evaluation, AI Safety, and AI System Design.", "topics": [ "ai-course", "ai-engineering", "ai-engineering-course", "ai-engineering-curriculum", "large-language-models" ], "category": "Red Teaming & Safety", "momentum": 74, "rank": 40 }, { "name": "AI-Engineer-Headquarters", "full_name": "hemansnation/AI-Engineer-Headquarters", "slug": "hemansnation-ai-engineer-headquarters", "url": "https://github.com/hemansnation/AI-Engineer-Headquarters", "owner": "hemansnation", "owner_avatar": "https://avatars.githubusercontent.com/u/37770869?v=4", "stars": 3777, "forks": 703, "open_issues": 17, "language": "Jupyter Notebook", "license": "", "pushed_at": "2026-10-28T13:49:57Z", "created_at": "2022-03-29T15:30:22Z", "description": "A collection of scientific methods, processes, algorithms, and systems to build stories & models.", "topics": [ "ai", "ai-agents", "aiengineering", "data-structures-and-algorithms", "deep-learning", "langchain", "langgraph", "llm", "llm-evaluation", "llm-inference", "llm-security", "machine-learning", "mlops", "natural-language-processing", "python", "pytorch", "rag", "statistics" ], "category": "RAG Eval", "momentum": 73, "rank": 41 }, { "name": "ChainForge", "full_name": "ianarawjo/ChainForge", "slug": "ianarawjo-chainforge", "url": "https://github.com/ianarawjo/ChainForge", "owner": "ianarawjo", "owner_avatar": "https://avatars.githubusercontent.com/u/5251713?v=4", "stars": 3040, "forks": 260, "open_issues": 70, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-10-29T23:54:02Z", "created_at": "2023-03-26T16:55:33Z", "description": "An open-source visual programming environment for battle-testing prompts to LLMs.", "topics": [ "ai", "evaluation", "large-language-models", "llmops", "llms", "prompt-engineering" ], "category": "Coding Eval", "momentum": 73, "rank": 42 }, { "name": "openlit", "full_name": "openlit/openlit", "slug": "openlit-openlit", "url": "https://github.com/openlit/openlit", "owner": "openlit", "owner_avatar": "https://avatars.githubusercontent.com/u/149867240?v=4", "stars": 2902, "forks": 456, "open_issues": 213, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2026-11-03T14:26:09Z", "created_at": "2024-01-23T17:40:59Z", "description": "OpenLIT is the open-source agent harness engineering platform: trace, evaluate, guard, and improve everything around the model in your AI agents, on OTEL.", "topics": [ "agent-harness", "agent-harness-engineering", "ai-engineering", "ai-observability", "amd-gpu", "claude-code", "clickhouse", "codex", "coding-agent", "coding-agents", "cursor", "genai", "gpu-monitoring", "llm-observability", "llmops", "nvidia-smi", "observability", "open-source", "openai", "opentelemetry" ], "category": "Observability", "momentum": 73, "rank": 43 }, { "name": "free-coding-models", "full_name": "vava-nessa/free-coding-models", "slug": "vava-nessa-free-coding-models", "url": "https://github.com/vava-nessa/free-coding-models", "owner": "vava-nessa", "owner_avatar": "https://avatars.githubusercontent.com/u/5466264?v=4", "stars": 2869, "forks": 297, "open_issues": 0, "language": "HTML", "license": "NOASSERTION", "pushed_at": "2026-11-03T14:56:47Z", "created_at": "2026-02-20T23:12:50Z", "description": "Find, benchmark and install in CLI 170+ FREE coding LLM models across 15+ providers in real time", "topics": [ "ai", "deepseek", "free", "free-ai", "freeai", "gpt", "gptoss", "kimi", "nim", "nvidia", "nvidia-nim", "nvidia-nim-api", "nvidia-nims", "openclaw", "opencode" ], "category": "Coding Eval", "momentum": 73, "rank": 44 }, { "name": "evidently", "full_name": "evidentlyai/evidently", "slug": "evidentlyai-evidently", "url": "https://github.com/evidentlyai/evidently", "owner": "evidentlyai", "owner_avatar": "https://avatars.githubusercontent.com/u/75031056?v=4", "stars": 8032, "forks": 964, "open_issues": 353, "language": "Jupyter Notebook", "license": "Apache-2.0", "pushed_at": "2026-09-29T08:32:43Z", "created_at": "2020-11-25T15:20:08Z", "description": "Evidently is ​​an open-source ML and LLM observability framework. Evaluate, test, and monitor any AI-powered system or data pipeline. From tabular data to Gen AI. 100+ metrics.", "topics": [ "data-drift", "data-quality", "data-science", "data-validation", "generative-ai", "hacktoberfest", "html-report", "jupyter-notebook", "llm", "llmops", "machine-learning", "mlops", "model-monitoring", "pandas-dataframe" ], "category": "Observability", "momentum": 72, "rank": 45 }, { "name": "lighteval", "full_name": "huggingface/lighteval", "slug": "huggingface-lighteval", "url": "https://github.com/huggingface/lighteval", "owner": "huggingface", "owner_avatar": "https://avatars.githubusercontent.com/u/25720743?v=4", "stars": 2578, "forks": 605, "open_issues": 480, "language": "Python", "license": "MIT", "pushed_at": "2026-10-29T14:15:10Z", "created_at": "2024-01-26T13:15:39Z", "description": "Lighteval is your all-in-one toolkit for evaluating LLMs across multiple backends", "topics": [ "evaluation", "evaluation-framework", "evaluation-metrics", "huggingface" ], "category": "Eval Frameworks", "momentum": 72, "rank": 46 }, { "name": "AssetOpsBench", "full_name": "IBM/AssetOpsBench", "slug": "ibm-assetopsbench", "url": "https://github.com/IBM/AssetOpsBench", "owner": "IBM", "owner_avatar": "https://avatars.githubusercontent.com/u/1459110?v=4", "stars": 2409, "forks": 359, "open_issues": 69, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T15:04:09Z", "created_at": "2025-05-01T20:48:50Z", "description": "AssetOpsBench - Industry 4.0: A unified benchmark and framework for building, orchestrating, and evaluating domain-specific AI agents for Industry 4.0 asset operations and maintenance, with 460+ scenarios, 5 specialist agents (IoT, FMSR, TSFM, Work Order,...), and multi-agent orchestration blueprints (MetaAgent, AgentHive) over MCP.", "topics": [ "ai-for-physical-assets", "condition-based-maintenance", "hvac-maintenance", "iot", "llm-agents", "model-context-protocol", "predictive-maintenance", "time-series-forecasting" ], "category": "Agent Eval", "momentum": 72, "rank": 47 }, { "name": "future-agi", "full_name": "future-agi/future-agi", "slug": "future-agi-future-agi", "url": "https://github.com/future-agi/future-agi", "owner": "future-agi", "owner_avatar": "https://avatars.githubusercontent.com/u/147392366?v=4", "stars": 2241, "forks": 704, "open_issues": 956, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:37:17Z", "created_at": "2026-04-23T14:11:31Z", "description": "Open-source, end-to-end platform for evaluating, observing, and improving LLM and AI agent applications. Tracing · Evals · Simulations · Datasets · Gateway · Guardrails. Self-hostable. Apache 2.0.", "topics": [ "agent-evaluation", "ai-agents", "ai-evals", "ai-gateway", "ai-observability", "evals", "evaluation-framework", "guardrails", "hallucination-detection", "llm-evaluation", "llm-observability", "llm-tracing", "llmops", "model-evaluation", "observability", "opentelemetry", "rag-evaluation", "simulation", "telemetry", "tracing" ], "category": "Observability", "momentum": 72, "rank": 48 }, { "name": "keystone-bench", "full_name": "xinxuxin/keystone-bench", "slug": "xinxuxin-keystone-bench", "url": "https://github.com/xinxuxin/keystone-bench", "owner": "xinxuxin", "owner_avatar": "https://avatars.githubusercontent.com/u/127476761?v=4", "stars": 1114, "forks": 3, "open_issues": 0, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-10-19T21:06:11Z", "created_at": "2026-09-07T17:45:56Z", "description": "A clinical decision-evidence benchmark: pull the one fact an answer rests on, and see whether the assistant's action moves with the evidence. Built on HealthBench.", "topics": [ "ai-safety", "benchmark", "clinical-decision-support", "dataset", "evaluation", "health", "healthbench", "large-language-models", "llm-evaluation", "medical-ai" ], "category": "Red Teaming & Safety", "momentum": 72, "rank": 49 }, { "name": "Cybermes", "full_name": "Zyrexnn/Cybermes", "slug": "zyrexnn-cybermes", "url": "https://github.com/Zyrexnn/Cybermes", "owner": "Zyrexnn", "owner_avatar": "https://avatars.githubusercontent.com/u/177414770?v=4", "stars": 999, "forks": 165, "open_issues": 8, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-01T21:15:25Z", "created_at": "2026-08-19T13:25:08Z", "description": "Autonomous Offensive Security, Bug Bounty & Red Teaming Agent Framework powered by Hermes Agent, specialized reasoning skills, and multi-model LLM orchestration.", "topics": [ "ai-agent", "appsec", "bug-bounty", "cvss-v3", "cybersecurity", "devsecops", "exploit-validation", "hermes-agent", "infosec", "llm-security", "nuclei", "offensive-security", "penetration-testing", "playwright", "reconnaissance", "red-teaming", "security-tools", "sqlmap", "threat-research", "vulnerability-scanner" ], "category": "Red Teaming & Safety", "momentum": 72, "rank": 50 }, { "name": "deepteam", "full_name": "confident-ai/deepteam", "slug": "confident-ai-deepteam", "url": "https://github.com/confident-ai/deepteam", "owner": "confident-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/130858411?v=4", "stars": 3085, "forks": 504, "open_issues": 82, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-23T11:32:04Z", "created_at": "2025-03-05T06:34:21Z", "description": "DeepTeam is a framework to red team LLMs and AI agents.", "topics": [ "llm-guardrails", "llm-red-teaming", "llm-safety", "llm-seecurity", "python" ], "category": "Red Teaming & Safety", "momentum": 71, "rank": 51 }, { "name": "agentic_security", "full_name": "msoedov/agentic_security", "slug": "msoedov-agentic-security", "url": "https://github.com/msoedov/agentic_security", "owner": "msoedov", "owner_avatar": "https://avatars.githubusercontent.com/u/1958116?v=4", "stars": 2040, "forks": 301, "open_issues": 13, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-01T18:34:04Z", "created_at": "2024-04-11T17:18:54Z", "description": "Agentic LLM Vulnerability Scanner / AI red teaming kit 🧪", "topics": [ "agent-framework", "agent-security", "ai-red-team", "llm-evaluation", "llm-evaluation-framework", "llm-fuzzer", "llm-fuzzer-aggregator", "llm-fuzzing", "llm-guardrails", "llm-jailbreaks", "llm-scanner", "llm-security", "llm-vulnerabilities", "prompt-testing" ], "category": "Red Teaming & Safety", "momentum": 71, "rank": 52 }, { "name": "trpc-agent-go", "full_name": "trpc-group/trpc-agent-go", "slug": "trpc-group-trpc-agent-go", "url": "https://github.com/trpc-group/trpc-agent-go", "owner": "trpc-group", "owner_avatar": "https://avatars.githubusercontent.com/u/140800716?v=4", "stars": 1898, "forks": 336, "open_issues": 181, "language": "Go", "license": "Apache-2.0", "pushed_at": "2026-11-03T07:56:11Z", "created_at": "2025-05-14T13:51:35Z", "description": "A Go framework for building production agent systems with graph workflows, tools, memory, A2A, AG-UI, MCP, evaluation, and observability.", "topics": [ "a2a", "a2a-protocol", "ag-ui", "agent", "agent-framework", "ai", "ai-agents", "evaluation", "go", "golang", "graph-workflows", "llm", "mcp", "model-context-protocol", "multi-agent", "observability", "opentelemetry", "rag" ], "category": "Observability", "momentum": 71, "rank": 53 }, { "name": "awesome-ai-agent-papers", "full_name": "VoltAgent/awesome-ai-agent-papers", "slug": "voltagent-awesome-ai-agent-papers", "url": "https://github.com/VoltAgent/awesome-ai-agent-papers", "owner": "VoltAgent", "owner_avatar": "https://avatars.githubusercontent.com/u/201282378?v=4", "stars": 1885, "forks": 196, "open_issues": 0, "language": "", "license": "MIT", "pushed_at": "2026-11-02T06:48:41Z", "created_at": "2026-02-10T10:58:31Z", "description": "A curated collection of AI agent research papers released in 2026, covering agent engineering, memory, evaluation, workflows, and autonomous systems.", "topics": [ "ai-agents", "awesome", "awesome-list", "llm", "llm-agents", "memory", "rag", "research-paper" ], "category": "RAG Eval", "momentum": 71, "rank": 54 }, { "name": "Introduction-to-Quantitative-Finance", "full_name": "Barca0412/Introduction-to-Quantitative-Finance", "slug": "barca0412-introduction-to-quantitative-finance", "url": "https://github.com/Barca0412/Introduction-to-Quantitative-Finance", "owner": "Barca0412", "owner_avatar": "https://avatars.githubusercontent.com/u/92302656?v=4", "stars": 1862, "forks": 193, "open_issues": 6, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T03:07:46Z", "created_at": "2023-08-04T04:10:47Z", "description": "AI+金融（量化）：1.多因子股票量化框架开源教程 2.学界和业界的经典资料收录 3.AI + 金融的相关工作，包括LLM, Agent, benchmark(evaluation), etc.", "topics": [ "agent", "ai4fin", "finance", "investment", "llm", "llm4fin", "quant", "quantitative-finance", "quantitative-research", "quantitative-trading", "trading" ], "category": "Agent Eval", "momentum": 71, "rank": 55 }, { "name": "WFGY", "full_name": "onestardao/WFGY", "slug": "onestardao-wfgy", "url": "https://github.com/onestardao/WFGY", "owner": "onestardao", "owner_avatar": "https://avatars.githubusercontent.com/u/212544643?v=4", "stars": 1804, "forks": 165, "open_issues": 14, "language": "Jupyter Notebook", "license": "NOASSERTION", "pushed_at": "2026-11-03T13:49:34Z", "created_at": "2025-06-04T13:45:14Z", "description": "WFGY is heading toward WFGY 5.0 Polaris Protocol, a major open-source release for AI reasoning, RAG, agents, and real-world workflows. Includes Problem Map, Global Debug Card, WFGY 4.0, and the CFV Easter Egg.", "topics": [ "ai-agents", "alignment", "debugging", "evaluation", "graphrag", "hallucination", "information-retrieval", "knowledge-graph", "llm", "rag", "reasoning", "retrieval-augmented-generation" ], "category": "RAG Eval", "momentum": 71, "rank": 56 }, { "name": "agy-staff", "full_name": "keli-wen/agy-staff", "slug": "keli-wen-agy-staff", "url": "https://github.com/keli-wen/agy-staff", "owner": "keli-wen", "owner_avatar": "https://avatars.githubusercontent.com/u/103916249?v=4", "stars": 815, "forks": 56, "open_issues": 4, "language": "JavaScript", "license": "MIT", "pushed_at": "2026-11-02T09:37:02Z", "created_at": "2026-08-18T16:08:25Z", "description": "Hire Google's Antigravity CLI (agy) as a fast Gemini staffer for Claude Code and OpenAI Codex.", "topics": [ "agent-evaluation", "agentic-ai", "ai-agents", "antigravity", "antigravity-cli", "benchmark", "claude-code", "claude-code-plugin", "code-review", "codex", "codex-plugin", "developer-tools", "gemini", "llm", "llm-evaluation", "multi-agent", "openai-codex", "opencode-plugin", "opencode-skills", "pi-skills" ], "category": "Agent Eval", "momentum": 71, "rank": 57 }, { "name": "awesome-ai-security-tools", "full_name": "scadastrangelove/awesome-ai-security-tools", "slug": "scadastrangelove-awesome-ai-security-tools", "url": "https://github.com/scadastrangelove/awesome-ai-security-tools", "owner": "scadastrangelove", "owner_avatar": "https://avatars.githubusercontent.com/u/16428031?v=4", "stars": 1713, "forks": 464, "open_issues": 14, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-01T07:15:21Z", "created_at": "2026-05-27T12:52:20Z", "description": "A curated list of public-source, research, and commercial tools for AI security and AI-assisted cybersecurity — autotriage, agent security, AI/ML supply chain, pentest agents, AI SAST, LLM-driven fuzzing, threat intelligence, SOC/SIEM triage, reverse engineering, LLM red-teaming, and more.", "topics": [ "agentic-ai", "agents", "awesome", "awesome-list", "awesome-lists", "llm", "security" ], "category": "Red Teaming & Safety", "momentum": 70, "rank": 58 }, { "name": "zilla", "full_name": "aklivity/zilla", "slug": "aklivity-zilla", "url": "https://github.com/aklivity/zilla", "owner": "aklivity", "owner_avatar": "https://avatars.githubusercontent.com/u/79664355?v=4", "stars": 1689, "forks": 146, "open_issues": 338, "language": "Java", "license": "NOASSERTION", "pushed_at": "2026-11-02T08:13:47Z", "created_at": "2021-12-07T22:10:04Z", "description": "🦎 A lightweight, streaming-native, multi-protocol gateway for AI, APIs, and real-time data. Route LLM requests across providers, govern MCP tools, and connect Kafka, MQTT, and HTTP through one high-performance engine with shared security and observability.", "topics": [ "api-gateway", "asyncapi", "event-driven-architecture", "event-stream-proxy", "event-streaming", "grpc", "iot", "kafka", "kafka-proxy", "mqtt", "openapi", "proxy", "rest", "server-sent-events", "streaming-api" ], "category": "Observability", "momentum": 70, "rank": 59 }, { "name": "Awesome-LLMs-for-Vulnerability-Detection", "full_name": "huhusmang/Awesome-LLMs-for-Vulnerability-Detection", "slug": "huhusmang-awesome-llms-for-vulnerability-detection", "url": "https://github.com/huhusmang/Awesome-LLMs-for-Vulnerability-Detection", "owner": "huhusmang", "owner_avatar": "https://avatars.githubusercontent.com/u/77958410?v=4", "stars": 1508, "forks": 154, "open_issues": 6, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T04:19:29Z", "created_at": "2024-09-02T13:49:55Z", "description": "The community's most comprehensive, continuously-updated index of research on Large Language Models for software vulnerability detection — papers across function-level, repository-level, agentic, and smart-contract detection, plus datasets, benchmarks, and surveys.", "topics": [ "awesome-list", "code-security", "large-language-models", "llm", "security", "static-analysis", "vulnerability-detection" ], "category": "Agent Eval", "momentum": 70, "rank": 60 }, { "name": "SkillCorpus", "full_name": "EverMind-AI/SkillCorpus", "slug": "evermind-ai-skillcorpus", "url": "https://github.com/EverMind-AI/SkillCorpus", "owner": "EverMind-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/229275294?v=4", "stars": 695, "forks": 115, "open_issues": 5, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T13:47:37Z", "created_at": "2026-08-11T02:48:55Z", "description": "Open-source infrastructure that turns scattered SKILL.md files into curated, retrieval-ready agent-skill corpora—with retrieval and evaluation tooling included.", "topics": [ "agent-memory", "agent-skills", "ai-agents", "benchmark", "dataset", "deepseek-harness", "dsh", "dsh-plugin", "embeddings", "hugginface", "information-retrieval", "llm-agents", "llm-evaluation", "long-term-memory", "python", "reranking", "semantic-search", "skill-md", "skill-routing", "vector-search" ], "category": "RAG Eval", "momentum": 70, "rank": 61 }, { "name": "llm-as-a-verifier", "full_name": "llm-as-a-verifier/llm-as-a-verifier", "slug": "llm-as-a-verifier-llm-as-a-verifier", "url": "https://github.com/llm-as-a-verifier/llm-as-a-verifier", "owner": "llm-as-a-verifier", "owner_avatar": "https://avatars.githubusercontent.com/u/272442170?v=4", "stars": 3344, "forks": 280, "open_issues": 26, "language": "Python", "license": "MIT", "pushed_at": "2026-10-08T23:45:12Z", "created_at": "2026-04-09T17:28:59Z", "description": "LLM-as-a-Verifier is a general-purpose framework that provides fine-grained feedback for any agent without requiring additional training. It achieves SOTA performance across coding, robotics, and medical agentic benchmarks.", "topics": [], "category": "Agent Eval", "momentum": 69, "rank": 62 }, { "name": "skillsbench", "full_name": "benchflow-ai/skillsbench", "slug": "benchflow-ai-skillsbench", "url": "https://github.com/benchflow-ai/skillsbench", "owner": "benchflow-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/190338344?v=4", "stars": 1895, "forks": 376, "open_issues": 122, "language": "PDDL", "license": "Apache-2.0", "pushed_at": "2026-10-24T05:35:25Z", "created_at": "2025-12-29T05:04:57Z", "description": "SkillsBench evaluates how well skills work and how effective agents are at using them.", "topics": [ "benchmark", "skillsbench" ], "category": "Agent Eval", "momentum": 69, "rank": 63 }, { "name": "LLM-eval-survey", "full_name": "MLGroupJLU/LLM-eval-survey", "slug": "mlgroupjlu-llm-eval-survey", "url": "https://github.com/MLGroupJLU/LLM-eval-survey", "owner": "MLGroupJLU", "owner_avatar": "https://avatars.githubusercontent.com/u/138345091?v=4", "stars": 1615, "forks": 106, "open_issues": 6, "language": "", "license": "", "pushed_at": "2026-10-28T15:45:03Z", "created_at": "2023-07-02T03:23:20Z", "description": "The official GitHub page for the survey paper \"A Survey on Evaluation of Large Language Models\".", "topics": [ "benchmark", "evaluation", "large-language-models", "llm", "llms", "model-assessment" ], "category": "Benchmarks", "momentum": 69, "rank": 64 }, { "name": "harvey-labs", "full_name": "harveyai/harvey-labs", "slug": "harveyai-harvey-labs", "url": "https://github.com/harveyai/harvey-labs", "owner": "harveyai", "owner_avatar": "https://avatars.githubusercontent.com/u/117767849?v=4", "stars": 1556, "forks": 291, "open_issues": 64, "language": "Python", "license": "MIT", "pushed_at": "2026-10-30T17:26:42Z", "created_at": "2026-03-30T18:20:43Z", "description": "A benchmark built to evaluate and improve agent capabilities for supporting legal work.", "topics": [], "category": "Agent Eval", "momentum": 69, "rank": 65 }, { "name": "Gym", "full_name": "NVIDIA-NeMo/Gym", "slug": "nvidia-nemo-gym", "url": "https://github.com/NVIDIA-NeMo/Gym", "owner": "NVIDIA-NeMo", "owner_avatar": "https://avatars.githubusercontent.com/u/213689629?v=4", "stars": 1314, "forks": 442, "open_issues": 1364, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:36:09Z", "created_at": "2025-08-25T21:37:55Z", "description": "Evaluate and improve models and agents using environments", "topics": [ "agents", "benchmarks", "environments", "evaluation", "gym", "llm", "reinforcement-learning", "reinforcement-learning-environments", "rl-environment", "rl-training" ], "category": "Agent Eval", "momentum": 69, "rank": 66 }, { "name": "prompty", "full_name": "microsoft/prompty", "slug": "microsoft-prompty", "url": "https://github.com/microsoft/prompty", "owner": "microsoft", "owner_avatar": "https://avatars.githubusercontent.com/u/6154722?v=4", "stars": 1286, "forks": 134, "open_issues": 24, "language": "Rust", "license": "MIT", "pushed_at": "2026-11-02T18:52:24Z", "created_at": "2024-04-22T16:55:53Z", "description": "Prompty makes it easy to create, manage, debug, and evaluate LLM prompts for your AI applications. Prompty is an asset class and format for LLM prompts designed to enhance observability, understandability, and portability for developers.", "topics": [ "generative-ai", "llm-evaluation", "llms", "promptengineering", "prompty" ], "category": "Observability", "momentum": 69, "rank": 67 }, { "name": "nautilus-compass", "full_name": "chunxiaoxx/nautilus-compass", "slug": "chunxiaoxx-nautilus-compass", "url": "https://github.com/chunxiaoxx/nautilus-compass", "owner": "chunxiaoxx", "owner_avatar": "https://avatars.githubusercontent.com/u/253869585?v=4", "stars": 1254, "forks": 35, "open_issues": 33, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-01T14:47:09Z", "created_at": "2026-04-27T18:20:31Z", "description": "Open memory & reliability layer for AI agents + independent judging/verification (NACRE judge, caliber-bench meta-benchmark, preregistered criteria) — by Nautilus Platform", "topics": [ "a2a", "agent-memory", "agentic-ai", "benchmark", "claude-code", "cross-agent", "evaluation", "llm", "llm-as-judge", "local-first", "long-term-memory", "mcp", "memory-layer", "nautilus", "rag", "verification" ], "category": "RAG Eval", "momentum": 68, "rank": 68 }, { "name": "Agent_Memory_Techniques", "full_name": "NirDiamant/Agent_Memory_Techniques", "slug": "nirdiamant-agent-memory-techniques", "url": "https://github.com/NirDiamant/Agent_Memory_Techniques", "owner": "NirDiamant", "owner_avatar": "https://avatars.githubusercontent.com/u/28316913?v=4", "stars": 1132, "forks": 141, "open_issues": 7, "language": "Jupyter Notebook", "license": "Apache-2.0", "pushed_at": "2026-11-01T13:57:49Z", "created_at": "2026-05-05T21:08:23Z", "description": "Agent memory for LLMs: 30 runnable Jupyter notebooks covering conversation buffers, vector stores, knowledge graphs, episodic and semantic memory, MemGPT, Mem0, Letta, Zep, Graphiti, LoCoMo benchmarks, and production patterns.", "topics": [ "agent-memory", "ai-agents", "anthropic", "episodic-memory", "generative-ai", "graphiti", "knowledge-graph", "langchain", "letta", "llm", "llm-agents", "llm-memory", "mem0", "memgpt", "openai", "python", "rag", "semantic-memory", "vector-database", "zep" ], "category": "RAG Eval", "momentum": 68, "rank": 69 }, { "name": "judgeval", "full_name": "JudgmentLabs/judgeval", "slug": "judgmentlabs-judgeval", "url": "https://github.com/JudgmentLabs/judgeval", "owner": "JudgmentLabs", "owner_avatar": "https://avatars.githubusercontent.com/u/186249420?v=4", "stars": 1074, "forks": 106, "open_issues": 49, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T21:01:23Z", "created_at": "2024-10-25T04:49:15Z", "description": "The Continuous-Improvement Stack for Agents. Our environment data and evals power agent improvement and monitoring.", "topics": [ "agent", "agentic-ai", "agents", "grpo", "langchain", "langgraph", "llama-index", "llm", "llm-evaluation", "llm-observability", "open-source", "openai", "prompt-engineering", "reinforcement-learning", "rl" ], "category": "Observability", "momentum": 68, "rank": 70 }, { "name": "OpenJudge", "full_name": "agentscope-ai/OpenJudge", "slug": "agentscope-ai-openjudge", "url": "https://github.com/agentscope-ai/OpenJudge", "owner": "agentscope-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/211762292?v=4", "stars": 1039, "forks": 99, "open_issues": 13, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T09:52:44Z", "created_at": "2025-07-08T06:17:01Z", "description": "OpenJudge: A Unified Framework for Holistic Evaluation and Quality Rewards", "topics": [ "agent", "agent-skills", "ai-agent", "alignment", "evaluation", "grader", "llm", "reward", "reward-model", "rlhf", "skill-md", "skills" ], "category": "Agent Eval", "momentum": 68, "rank": 71 }, { "name": "ClawBench", "full_name": "TIGER-AI-Lab/ClawBench", "slug": "tiger-ai-lab-clawbench", "url": "https://github.com/TIGER-AI-Lab/ClawBench", "owner": "TIGER-AI-Lab", "owner_avatar": "https://avatars.githubusercontent.com/u/144196744?v=4", "stars": 1193, "forks": 74, "open_issues": 100, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-23T15:52:23Z", "created_at": "2026-04-10T01:59:17Z", "description": "Open-source benchmark for browser AI agents on daily tasks.", "topics": [ "agent-evaluation", "agentic-ai", "ai-agent-benchmark", "ai-agents", "benchmark", "browser-agent", "browser-automation", "browser-use", "chrome-agent", "chrome-extension", "computer-use", "dataset", "evaluation", "everyday-tasks", "llm", "llm-evaluation", "online-tasks", "real-world-benchmark", "web-agent", "web-agents" ], "category": "Agent Eval", "momentum": 67, "rank": 72 }, { "name": "audio-ai-hub", "full_name": "BinWang28/audio-ai-hub", "slug": "binwang28-audio-ai-hub", "url": "https://github.com/BinWang28/audio-ai-hub", "owner": "BinWang28", "owner_avatar": "https://avatars.githubusercontent.com/u/25280416?v=4", "stars": 962, "forks": 57, "open_issues": 142, "language": "Python", "license": "", "pushed_at": "2026-11-02T12:36:40Z", "created_at": "2024-06-15T06:26:43Z", "description": "The hub for audio AI research: papers, open models, benchmarks & datasets across audio LLMs, speech recognition, TTS, music & audio generation.", "topics": [ "audio-ai", "audio-generation", "audio-llm", "music-generation", "paper-list", "speech-recognition", "tts" ], "category": "Benchmarks", "momentum": 67, "rank": 73 }, { "name": "fable-method", "full_name": "Sahir619/fable-method", "slug": "sahir619-fable-method", "url": "https://github.com/Sahir619/fable-method", "owner": "Sahir619", "owner_avatar": "https://avatars.githubusercontent.com/u/94783820?v=4", "stars": 2313, "forks": 325, "open_issues": 4, "language": "Python", "license": "MIT", "pushed_at": "2026-10-03T00:38:29Z", "created_at": "2026-07-06T22:35:18Z", "description": "The Fable Workflow: how Claude Fable 5 worked, distilled into skills any model can run, with the eval that keeps it honest. Think / act / prove.", "topics": [ "agent-skills", "ai-agents", "claude", "claude-code", "claude-md", "coding-agent", "evaluation", "fable", "llm" ], "category": "Agent Eval", "momentum": 66, "rank": 74 }, { "name": "LiveBench", "full_name": "LiveBench/LiveBench", "slug": "livebench-livebench", "url": "https://github.com/LiveBench/LiveBench", "owner": "LiveBench", "owner_avatar": "https://avatars.githubusercontent.com/u/169931984?v=4", "stars": 1352, "forks": 130, "open_issues": 186, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-10-18T01:48:44Z", "created_at": "2024-06-12T12:13:57Z", "description": "LiveBench: A Challenging, Contamination-Free LLM Benchmark", "topics": [], "category": "Benchmarks", "momentum": 66, "rank": 75 }, { "name": "awesome-evals", "full_name": "benchflow-ai/awesome-evals", "slug": "benchflow-ai-awesome-evals", "url": "https://github.com/benchflow-ai/awesome-evals", "owner": "benchflow-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/190338344?v=4", "stars": 1023, "forks": 143, "open_issues": 77, "language": "", "license": "NOASSERTION", "pushed_at": "2026-10-27T18:50:51Z", "created_at": "2026-06-24T08:10:33Z", "description": "A curated, non-BS library of the best resources for building and evaluating AI agents — papers, blogs, talks, tools, benchmarks. Maintained by BenchFlow.", "topics": [ "agent-evaluation", "ai-agents", "awesome", "awesome-list", "benchmarks", "evals", "llm", "llm-evaluation", "rl-environments" ], "category": "Agent Eval", "momentum": 66, "rank": 76 }, { "name": "overmind", "full_name": "overmind-core/overmind", "slug": "overmind-core-overmind", "url": "https://github.com/overmind-core/overmind", "owner": "overmind-core", "owner_avatar": "https://avatars.githubusercontent.com/u/208257599?v=4", "stars": 745, "forks": 75, "open_issues": 3, "language": "Python", "license": "AGPL-3.0", "pushed_at": "2026-11-03T13:40:17Z", "created_at": "2026-03-27T14:59:56Z", "description": "The platform for continuously improving AI agents.", "topics": [ "ai-agents", "evals", "fine-tuning", "llm", "llm-evaluation", "llmops", "lora", "mcp", "observability", "python" ], "category": "Observability", "momentum": 66, "rank": 77 }, { "name": "uqlm", "full_name": "cvs-health/uqlm", "slug": "cvs-health-uqlm", "url": "https://github.com/cvs-health/uqlm", "owner": "cvs-health", "owner_avatar": "https://avatars.githubusercontent.com/u/89528242?v=4", "stars": 1217, "forks": 140, "open_issues": 26, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-13T20:14:59Z", "created_at": "2025-04-17T23:12:21Z", "description": "[JMLR 2026] UQLM: A Python package for LLM hallucination detection using uncertainty quantification", "topics": [ "ai-evaluation", "ai-safety", "confidence-estimation", "confidence-score", "hallucination", "hallucination-detection", "hallucination-evaluation", "hallucination-mitigation", "llm", "llm-evaluation", "llm-hallucination", "llm-safety", "uncertainty-estimation", "uncertainty-quantification" ], "category": "Red Teaming & Safety", "momentum": 65, "rank": 78 }, { "name": "inspect-robots", "full_name": "robocurve/inspect-robots", "slug": "robocurve-inspect-robots", "url": "https://github.com/robocurve/inspect-robots", "owner": "robocurve", "owner_avatar": "https://avatars.githubusercontent.com/u/297012305?v=4", "stars": 726, "forks": 96, "open_issues": 84, "language": "Python", "license": "MIT", "pushed_at": "2026-10-29T04:53:09Z", "created_at": "2026-06-26T06:55:29Z", "description": "Open source evals for physical AI. Run any LLM/VLA on any arm/humanoid against any real/sim benchmark.", "topics": [ "benchmark", "embodied-ai", "evaluation", "physical-ai", "rerun", "robotics", "vision-language-action", "vla" ], "category": "Benchmarks", "momentum": 65, "rank": 79 }, { "name": "ParseBench", "full_name": "run-llama/ParseBench", "slug": "run-llama-parsebench", "url": "https://github.com/run-llama/ParseBench", "owner": "run-llama", "owner_avatar": "https://avatars.githubusercontent.com/u/130722866?v=4", "stars": 625, "forks": 132, "open_issues": 37, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T07:42:10Z", "created_at": "2026-04-10T20:46:33Z", "description": "ParseBench - A Document Parsing Benchmark for AI Agents", "topics": [ "benchmark", "document-ai", "document-parsing", "evaluation", "llamaindex", "llm", "machine-learning", "ocr", "pdf-parsing", "table-extraction", "vision-language-models" ], "category": "Agent Eval", "momentum": 65, "rank": 80 }, { "name": "XRAG", "full_name": "DocAILab/XRAG", "slug": "docailab-xrag", "url": "https://github.com/DocAILab/XRAG", "owner": "DocAILab", "owner_avatar": "https://avatars.githubusercontent.com/u/142860808?v=4", "stars": 611, "forks": 17, "open_issues": 1, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-01T09:44:03Z", "created_at": "2024-09-07T11:31:02Z", "description": "XRAG: eXamining the Core - Benchmarking Foundational Component Modules in Advanced Retrieval-Augmented Generation", "topics": [ "evaluation", "llamaindex", "llm", "rag" ], "category": "RAG Eval", "momentum": 65, "rank": 81 }, { "name": "PAKGOV-RAG-project", "full_name": "salikhussain71-code/PAKGOV-RAG-project", "slug": "salikhussain71-code-pakgov-rag-project", "url": "https://github.com/salikhussain71-code/PAKGOV-RAG-project", "owner": "salikhussain71-code", "owner_avatar": "https://avatars.githubusercontent.com/u/271632164?v=4", "stars": 130, "forks": 5, "open_issues": 2, "language": "Python", "license": "MIT", "pushed_at": "2026-11-02T12:36:23Z", "created_at": "2026-09-17T10:31:00Z", "description": "Research project on bilingual Urdu-English retrieval-augmented generation for Pakistani government and legal documents, with OCR, hybrid retrieval, citation grounding, and evaluation.", "topics": [ "artificial-intelligence", "information-retrieval", "legal-nlp", "low-resource-languages", "nlp", "ocr", "rag", "retrieval-augmented-generation", "transformers", "urdu-nlp" ], "category": "RAG Eval", "momentum": 65, "rank": 82 }, { "name": "agent-skills-eval", "full_name": "darkrishabh/agent-skills-eval", "slug": "darkrishabh-agent-skills-eval", "url": "https://github.com/darkrishabh/agent-skills-eval", "owner": "darkrishabh", "owner_avatar": "https://avatars.githubusercontent.com/u/812474?v=4", "stars": 809, "forks": 45, "open_issues": 7, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-10-21T15:24:05Z", "created_at": "2026-05-06T05:10:29Z", "description": "A test runner for agentskills.io-style AI agent skills", "topics": [ "agent-evals", "agent-skills", "agentskills", "ai-agents", "cli", "jsonl", "llm-evals", "llm-evaluation", "openai-compatible", "typescript", "yaml" ], "category": "Agent Eval", "momentum": 64, "rank": 83 }, { "name": "promptimize", "full_name": "preset-io/promptimize", "slug": "preset-io-promptimize", "url": "https://github.com/preset-io/promptimize", "owner": "preset-io", "owner_avatar": "https://avatars.githubusercontent.com/u/45908173?v=4", "stars": 497, "forks": 43, "open_issues": 17, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T22:34:53Z", "created_at": "2023-04-04T15:34:08Z", "description": "Promptimize is a prompt engineering evaluation and testing toolkit.", "topics": [], "category": "Eval Frameworks", "momentum": 64, "rank": 84 }, { "name": "awesome-MLSecOps", "full_name": "RiccardoBiosas/awesome-MLSecOps", "slug": "riccardobiosas-awesome-mlsecops", "url": "https://github.com/RiccardoBiosas/awesome-MLSecOps", "owner": "RiccardoBiosas", "owner_avatar": "https://avatars.githubusercontent.com/u/65150720?v=4", "stars": 485, "forks": 128, "open_issues": 27, "language": "Astro", "license": "MIT", "pushed_at": "2026-11-03T11:30:05Z", "created_at": "2023-04-01T10:06:32Z", "description": "A curated list of MLSecOps tools and resources for securing machine learning and AI systems - adversarial ML defense, LLM security, AI red teaming, model scanning, supply-chain protection, and MLOps pipeline security.", "topics": [ "adversarial-machine-learning", "agentic-security", "ai-agents", "ai-governance", "ai-red-teaming", "ai-security", "data-poisoning", "large-language-model", "llm-security", "machine-learning-security", "mcp-security", "mlops", "mlops-security", "mlsecops", "model-poisoning", "owasp", "prompt-injection", "rag-security", "supply-chain-security", "threat-modeling" ], "category": "Red Teaming & Safety", "momentum": 64, "rank": 85 }, { "name": "rag-fusion", "full_name": "Raudaschl/rag-fusion", "slug": "raudaschl-rag-fusion", "url": "https://github.com/Raudaschl/rag-fusion", "owner": "Raudaschl", "owner_avatar": "https://avatars.githubusercontent.com/u/6103284?v=4", "stars": 961, "forks": 113, "open_issues": 0, "language": "Python", "license": "MIT", "pushed_at": "2026-10-10T18:54:43Z", "created_at": "2023-09-25T21:58:31Z", "description": "RAG-Fusion: multi-query generation + Reciprocal Rank Fusion for better retrieval-augmented generation. Includes evaluation harness with NFCorpus/BEIR.", "topics": [ "chromadb", "information-retrieval", "openai", "python", "rag", "rag-fusion", "reciprocal-rank-fusion", "retrieval-augmented-generation", "vector-search" ], "category": "RAG Eval", "momentum": 63, "rank": 86 }, { "name": "RAG-FiT", "full_name": "IntelLabs/RAG-FiT", "slug": "intellabs-rag-fit", "url": "https://github.com/IntelLabs/RAG-FiT", "owner": "IntelLabs", "owner_avatar": "https://avatars.githubusercontent.com/u/1492758?v=4", "stars": 765, "forks": 61, "open_issues": 2, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-15T16:22:53Z", "created_at": "2024-07-23T13:20:35Z", "description": "Framework for enhancing LLMs for RAG tasks using fine-tuning.", "topics": [ "evaluation", "fine-tuning", "information-retrieval", "llm", "nlp", "question-answering", "rag", "semantic-search" ], "category": "RAG Eval", "momentum": 63, "rank": 87 }, { "name": "helm", "full_name": "stanford-crfm/helm", "slug": "stanford-crfm-helm", "url": "https://github.com/stanford-crfm/helm", "owner": "stanford-crfm", "owner_avatar": "https://avatars.githubusercontent.com/u/75054807?v=4", "stars": 2964, "forks": 436, "open_issues": 120, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-09-01T01:33:19Z", "created_at": "2021-11-29T08:53:17Z", "description": "Holistic Evaluation of Language Models (HELM) is an open source Python framework created by the Center for Research on Foundation Models (CRFM) at Stanford for holistic, reproducible and transparent evaluation of foundation models, including large language models (LLMs) and multimodal models.", "topics": [], "category": "Eval Frameworks", "momentum": 62, "rank": 88 }, { "name": "PIXIU", "full_name": "The-FinAI/PIXIU", "slug": "the-finai-pixiu", "url": "https://github.com/The-FinAI/PIXIU", "owner": "The-FinAI", "owner_avatar": "https://avatars.githubusercontent.com/u/26429143?v=4", "stars": 895, "forks": 126, "open_issues": 13, "language": "Jupyter Notebook", "license": "MIT", "pushed_at": "2026-10-08T12:20:27Z", "created_at": "2023-06-02T02:59:15Z", "description": "This repository introduces PIXIU, an open-source resource featuring the first financial large language models (LLMs), instruction tuning data, and evaluation benchmarks to holistically assess financial LLMs. Our goal is to continually push forward the open-source development of financial artificial intelligence (AI).", "topics": [ "aifinance", "chatgpt", "fintech", "gpt-4", "large-language-models", "llama", "machine-learning", "named-entity-recognition", "natural-language-processing", "nlp", "pixiu", "question-answering", "sentiment-analysis", "stock-price-prediction", "text-classification" ], "category": "Benchmarks", "momentum": 62, "rank": 89 }, { "name": "verifywise", "full_name": "verifywise-ai/verifywise", "slug": "verifywise-ai-verifywise", "url": "https://github.com/verifywise-ai/verifywise", "owner": "verifywise-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/262239808?v=4", "stars": 373, "forks": 127, "open_issues": 44, "language": "TypeScript", "license": "NOASSERTION", "pushed_at": "2026-11-03T11:36:19Z", "created_at": "2024-08-18T23:02:10Z", "description": "Complete AI governance and LLM Evals platform with support for EU AI Act, ISO 42001, NIST AI RMF and 20+ more AI frameworks and regulations. Join our Discord channel: https://discord.com/invite/d3k3E4uEpR", "topics": [ "ai", "ai-auditing", "ai-compliance", "ai-governance", "ai-governance-model", "ai-risk", "audit", "auditing", "compliance", "eu-ai-act", "governance", "grc", "iso27001", "iso42001", "llm-eval", "llm-evaluation", "nist-ai-rmf", "risk-management" ], "category": "Eval Frameworks", "momentum": 62, "rank": 90 }, { "name": "ai-rag-chat-evaluator", "full_name": "Azure-Samples/ai-rag-chat-evaluator", "slug": "azure-samples-ai-rag-chat-evaluator", "url": "https://github.com/Azure-Samples/ai-rag-chat-evaluator", "owner": "Azure-Samples", "owner_avatar": "https://avatars.githubusercontent.com/u/1844662?v=4", "stars": 336, "forks": 116, "open_issues": 25, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T08:13:29Z", "created_at": "2023-11-21T00:47:02Z", "description": "Tools for evaluation of RAG Chat Apps using Azure AI Evaluate SDK and OpenAI", "topics": [], "category": "RAG Eval", "momentum": 62, "rank": 91 }, { "name": "UltraEval-Audio", "full_name": "OpenBMB/UltraEval-Audio", "slug": "openbmb-ultraeval-audio", "url": "https://github.com/OpenBMB/UltraEval-Audio", "owner": "OpenBMB", "owner_avatar": "https://avatars.githubusercontent.com/u/89920203?v=4", "stars": 328, "forks": 29, "open_issues": 6, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T11:29:46Z", "created_at": "2024-11-11T09:41:43Z", "description": "Your faithful, impartial partner for audio evaluation — know yourself, know your rivals. 真实评测，知己知彼。A unified benchmark framework for ASR/TTS/Audio Codec/audio LLM evaluation", "topics": [ "asr", "audio-codec", "benchmark", "evaluation", "llm-evaluation", "speech-recognition", "speech-to-speech", "speech-to-text", "tts", "voice-cloning" ], "category": "Benchmarks", "momentum": 62, "rank": 92 }, { "name": "llmio", "full_name": "atopos31/llmio", "slug": "atopos31-llmio", "url": "https://github.com/atopos31/llmio", "owner": "atopos31", "owner_avatar": "https://avatars.githubusercontent.com/u/113456534?v=4", "stars": 322, "forks": 44, "open_issues": 11, "language": "Go", "license": "MIT", "pushed_at": "2026-10-29T17:52:09Z", "created_at": "2025-08-06T03:43:36Z", "description": "Unified LLM gateway with weighted load balancing, observability & cost tracking. 统一的 LLM 网关，提供权重负载均衡、可观测性与费用追踪。", "topics": [ "ai", "ai-gateway", "claude", "claude-ai", "claude-code", "codex", "deepseek", "doubao", "gemini", "gemini-cli", "go", "golang", "kimi", "newapi", "openai", "openclaw", "qwen", "transformer" ], "category": "Observability", "momentum": 61, "rank": 93 }, { "name": "rag-experiment-accelerator", "full_name": "microsoft/rag-experiment-accelerator", "slug": "microsoft-rag-experiment-accelerator", "url": "https://github.com/microsoft/rag-experiment-accelerator", "owner": "microsoft", "owner_avatar": "https://avatars.githubusercontent.com/u/6154722?v=4", "stars": 315, "forks": 112, "open_issues": 157, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-02T00:29:30Z", "created_at": "2023-09-25T16:47:36Z", "description": "The RAG Experiment Accelerator is a versatile tool designed to expedite and facilitate the process of conducting experiments and evaluations using Azure Cognitive Search and RAG pattern.", "topics": [ "acs", "azure", "chunking", "dense", "embedding", "evaluation", "experiment", "genai", "indexing", "information-retrieval", "llm", "openai", "rag", "sparse", "vectors" ], "category": "RAG Eval", "momentum": 61, "rank": 94 }, { "name": "awesome-llm-unlearning", "full_name": "chrisliu298/awesome-llm-unlearning", "slug": "chrisliu298-awesome-llm-unlearning", "url": "https://github.com/chrisliu298/awesome-llm-unlearning", "owner": "chrisliu298", "owner_avatar": "https://avatars.githubusercontent.com/u/59010212?v=4", "stars": 638, "forks": 35, "open_issues": 1, "language": "", "license": "Apache-2.0", "pushed_at": "2026-10-06T07:46:22Z", "created_at": "2024-03-27T15:44:23Z", "description": "A resource repository for machine unlearning in large language models", "topics": [ "ai-safety", "alignment", "awesome", "awesome-list", "evaluation", "knowledge-erasure", "large-language-model", "llm", "llm-safety", "llm-unlearning", "machine-learning", "machine-unlearning", "model-editing", "nlp", "paper-list", "privacy", "responsible-ai", "right-to-be-forgotten", "selective-forgetting", "unlearning" ], "category": "Red Teaming & Safety", "momentum": 60, "rank": 95 }, { "name": "TrustLLM", "full_name": "HowieHwong/TrustLLM", "slug": "howiehwong-trustllm", "url": "https://github.com/HowieHwong/TrustLLM", "owner": "HowieHwong", "owner_avatar": "https://avatars.githubusercontent.com/u/88385422?v=4", "stars": 635, "forks": 69, "open_issues": 8, "language": "Python", "license": "MIT", "pushed_at": "2026-10-04T23:28:06Z", "created_at": "2023-12-23T18:32:57Z", "description": "[ICML 2024] TrustLLM: LLM trustworthiness evaluation across truthfulness, safety, fairness, robustness, privacy and ethics. Python/CLI toolkit for local models, compatible APIs and AI agent orchestration.", "topics": [ "ai", "ai-agents", "benchmark", "dataset", "evaluation", "hallucination", "large-language-models", "llm", "llm-benchmark", "llm-evaluation", "llm-safety", "natural-language-processing", "nlp", "openai-compatible", "pypi-package", "responsible-ai", "toolkit", "trustworthy-ai", "trustworthy-machine-learning" ], "category": "Red Teaming & Safety", "momentum": 60, "rank": 96 }, { "name": "redcell", "full_name": "martian56/redcell", "slug": "martian56-redcell", "url": "https://github.com/martian56/redcell", "owner": "martian56", "owner_avatar": "https://avatars.githubusercontent.com/u/150589141?v=4", "stars": 354, "forks": 73, "open_issues": 4, "language": "Python", "license": "MIT", "pushed_at": "2026-10-06T12:16:48Z", "created_at": "2026-08-02T10:49:51Z", "description": "AI red-team platform. Autonomous LLM agents run a penetration test end to end inside a Kali container and write the report. LangGraph plan/act engine, provider-agnostic models via LiteLLM, PDF/JSON/SARIF output. FastAPI + React.", "topics": [ "ai-agents", "autonomous-agents", "cybersecurity", "fastapi", "langgraph", "litellm", "llm", "offensive-security", "penetration-testing", "pentesting", "python", "react", "red-team", "security", "security-tools", "typescript" ], "category": "Red Teaming & Safety", "momentum": 60, "rank": 97 }, { "name": "OpenHunterAI", "full_name": "LumosLab-Innovation/OpenHunterAI", "slug": "lumoslab-innovation-openhunterai", "url": "https://github.com/LumosLab-Innovation/OpenHunterAI", "owner": "LumosLab-Innovation", "owner_avatar": "https://avatars.githubusercontent.com/u/289335884?v=4", "stars": 338, "forks": 62, "open_issues": 1, "language": "TypeScript", "license": "NOASSERTION", "pushed_at": "2026-10-23T19:19:50Z", "created_at": "2026-05-25T17:58:49Z", "description": "Local-first AI red team for web, API, and LLM application security. Attacker-style reasoning, evidence-backed findings, and skills for AI coding agents.", "topics": [ "ai-security", "application-security", "appsec", "golang", "owasp", "pentesting", "playwright", "security-testing", "typescript", "web-security" ], "category": "Red Teaming & Safety", "momentum": 60, "rank": 98 }, { "name": "memorybench", "full_name": "supermemoryai/memorybench", "slug": "supermemoryai-memorybench", "url": "https://github.com/supermemoryai/memorybench", "owner": "supermemoryai", "owner_avatar": "https://avatars.githubusercontent.com/u/171979587?v=4", "stars": 334, "forks": 80, "open_issues": 42, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-10-24T17:50:39Z", "created_at": "2025-09-17T01:07:12Z", "description": "Unified benchmark for evaluating conversational memory and RAG across multiple datasets", "topics": [ "ai", "ai-memory", "benchmark-framework" ], "category": "RAG Eval", "momentum": 60, "rank": 99 }, { "name": "evals", "full_name": "strands-agents/evals", "slug": "strands-agents-evals", "url": "https://github.com/strands-agents/evals", "owner": "strands-agents", "owner_avatar": "https://avatars.githubusercontent.com/u/209155962?v=4", "stars": 250, "forks": 78, "open_issues": 136, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-30T17:36:15Z", "created_at": "2025-07-31T21:53:35Z", "description": "A comprehensive evaluation framework for AI agents and LLM applications.", "topics": [ "agentic", "agentic-ai", "ai", "evaluation", "machine-learning", "python", "strands-agents" ], "category": "Agent Eval", "momentum": 60, "rank": 100 }, { "name": "langfuse-docs", "full_name": "langfuse/langfuse-docs", "slug": "langfuse-langfuse-docs", "url": "https://github.com/langfuse/langfuse-docs", "owner": "langfuse", "owner_avatar": "https://avatars.githubusercontent.com/u/134601687?v=4", "stars": 249, "forks": 333, "open_issues": 256, "language": "MDX", "license": "MIT", "pushed_at": "2026-11-03T16:35:52Z", "created_at": "2023-05-22T14:48:56Z", "description": "🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.", "topics": [ "agent-skills", "agents", "genai", "hacktoberfest", "langfuse", "large-language-models", "llms", "nextra", "observability", "prompt-management" ], "category": "Observability", "momentum": 60, "rank": 101 }, { "name": "aisix", "full_name": "api7/aisix", "slug": "api7-aisix", "url": "https://github.com/api7/aisix", "owner": "api7", "owner_avatar": "https://avatars.githubusercontent.com/u/61078451?v=4", "stars": 212, "forks": 48, "open_issues": 68, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-11-03T16:00:23Z", "created_at": "2026-04-17T03:44:47Z", "description": "Open-source AI gateway for LLMs & AI agents, built in Rust. One OpenAI-compatible API for OpenAI, Anthropic, Gemini, Bedrock & more — routing, guardrails, caching, rate limits, observability.", "topics": [ "ai-agents", "ai-gateway", "ai-gateway-rust", "aisix", "anthropic", "bedrock", "gateway", "gemini", "guardrails", "inference-gateway", "llm", "llm-gateway", "llm-proxy", "llm-router", "llmops", "observability", "openai", "openai-compatible", "rust", "self-hosted" ], "category": "Observability", "momentum": 60, "rank": 102 }, { "name": "agent-starter-pack", "full_name": "GoogleCloudPlatform/agent-starter-pack", "slug": "googlecloudplatform-agent-starter-pack", "url": "https://github.com/GoogleCloudPlatform/agent-starter-pack", "owner": "GoogleCloudPlatform", "owner_avatar": "https://avatars.githubusercontent.com/u/2810941?v=4", "stars": 6576, "forks": 1497, "open_issues": 50, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-07-21T23:42:40Z", "created_at": "2025-01-31T16:03:06Z", "description": "Ship AI Agents to Google Cloud in minutes, not months. Production-ready templates with built-in CI/CD, evaluation, and observability.", "topics": [ "agents", "gcp", "gemini", "genai-agents", "generative-ai", "llmops", "mlops", "observability" ], "category": "Observability", "momentum": 59, "rank": 103 }, { "name": "yao-meta-skill", "full_name": "yaojingang/yao-meta-skill", "slug": "yaojingang-yao-meta-skill", "url": "https://github.com/yaojingang/yao-meta-skill", "owner": "yaojingang", "owner_avatar": "https://avatars.githubusercontent.com/u/266242792?v=4", "stars": 2719, "forks": 252, "open_issues": 4, "language": "Python", "license": "MIT", "pushed_at": "2026-08-17T09:19:43Z", "created_at": "2026-03-31T11:59:35Z", "description": "YAO = Yielding AI Outcomes. A rigorous engineering, evaluation, governance, and portability system for reusable agent skills.", "topics": [ "agent-skills", "ai-agents", "evaluation", "governance", "meta-skill", "portability", "skill-engineering", "workflow-automation" ], "category": "Agent Eval", "momentum": 59, "rank": 104 }, { "name": "Video-ChatGPT", "full_name": "mbzuai-oryx/Video-ChatGPT", "slug": "mbzuai-oryx-video-chatgpt", "url": "https://github.com/mbzuai-oryx/Video-ChatGPT", "owner": "mbzuai-oryx", "owner_avatar": "https://avatars.githubusercontent.com/u/133966423?v=4", "stars": 1515, "forks": 131, "open_issues": 28, "language": "Python", "license": "CC-BY-4.0", "pushed_at": "2026-09-05T16:57:07Z", "created_at": "2023-05-18T12:13:16Z", "description": "[ACL 2024 🔥] Video-ChatGPT is a video conversation model capable of generating meaningful conversation about videos. It combines the capabilities of LLMs with a pretrained visual encoder adapted for spatiotemporal video representation. We also introduce a rigorous 'Quantitative Evaluation Benchmarking' for video-based conversational models.", "topics": [ "chatbot", "clip", "gpt-4", "llama", "llava", "mulit-modal", "vicuna", "video-chatboat", "video-conversation", "vision-language", "vision-language-pretraining" ], "category": "Benchmarks", "momentum": 59, "rank": 105 }, { "name": "meta-agents-research-environments", "full_name": "facebookresearch/meta-agents-research-environments", "slug": "facebookresearch-meta-agents-research-environments", "url": "https://github.com/facebookresearch/meta-agents-research-environments", "owner": "facebookresearch", "owner_avatar": "https://avatars.githubusercontent.com/u/16943930?v=4", "stars": 577, "forks": 83, "open_issues": 49, "language": "Python", "license": "MIT", "pushed_at": "2026-09-30T22:56:10Z", "created_at": "2025-09-11T10:00:28Z", "description": "Meta Agents Research Environments is a comprehensive platform designed to evaluate AI agents in dynamic, realistic scenarios. Unlike static benchmarks, this platform introduces evolving environments where agents must adapt their strategies as new information becomes available, mirroring real-world challenges.", "topics": [ "agents", "ai", "autonomous-agents", "benchmark", "evaluation", "large-language-models", "llm", "meta", "multi-agent-systems", "natural-language-processing", "reinforcement-learning", "rl", "simulation" ], "category": "Agent Eval", "momentum": 59, "rank": 106 }, { "name": "open-rag-eval", "full_name": "vectara/open-rag-eval", "slug": "vectara-open-rag-eval", "url": "https://github.com/vectara/open-rag-eval", "owner": "vectara", "owner_avatar": "https://avatars.githubusercontent.com/u/108304503?v=4", "stars": 419, "forks": 30, "open_issues": 18, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-08T18:51:33Z", "created_at": "2024-11-26T19:32:17Z", "description": "RAG evaluation without the need for \"golden answers\"", "topics": [ "evaluation-metrics", "metrics", "rag", "rag-evaluation", "retrieval-augmented-generation", "vectara" ], "category": "RAG Eval", "momentum": 59, "rank": 107 }, { "name": "AgentPoison", "full_name": "AI-secure/AgentPoison", "slug": "ai-secure-agentpoison", "url": "https://github.com/AI-secure/AgentPoison", "owner": "AI-secure", "owner_avatar": "https://avatars.githubusercontent.com/u/61308241?v=4", "stars": 254, "forks": 35, "open_issues": 3, "language": "Python", "license": "MIT", "pushed_at": "2026-10-23T09:01:30Z", "created_at": "2024-03-22T10:39:10Z", "description": "[NeurIPS 2024] Official implementation for \"AgentPoison: Red-teaming LLM Agents via Memory or Knowledge Base Backdoor Poisoning\"", "topics": [ "llm-agent", "red-team", "retrieval-augmented-generation" ], "category": "Red Teaming & Safety", "momentum": 59, "rank": 108 }, { "name": "opik-mcp", "full_name": "comet-ml/opik-mcp", "slug": "comet-ml-opik-mcp", "url": "https://github.com/comet-ml/opik-mcp", "owner": "comet-ml", "owner_avatar": "https://avatars.githubusercontent.com/u/31487821?v=4", "stars": 225, "forks": 37, "open_issues": 7, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-28T10:45:52Z", "created_at": "2025-03-11T16:31:03Z", "description": "Model Context Protocol (MCP) server for Opik, the open-source LLM observability and evaluation platform, built by Comet. Read traces, log scores, and manage prompts from Claude Code, Cursor, or VS Code.", "topics": [ "claude-code", "generative-ai", "llm-observability", "mcp", "mcp-server", "model-context-protocol", "opik", "python" ], "category": "Observability", "momentum": 59, "rank": 109 }, { "name": "Argus", "full_name": "gy15901580825/Argus", "slug": "gy15901580825-argus", "url": "https://github.com/gy15901580825/Argus", "owner": "gy15901580825", "owner_avatar": "https://avatars.githubusercontent.com/u/59456306?v=4", "stars": 206, "forks": 28, "open_issues": 4, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T14:37:36Z", "created_at": "2026-05-28T04:30:38Z", "description": "Black-box, open-source red-team testing for AI agents. Point it at any HTTP, gRPC, or browser-using agent endpoint; run 205 probes mapped to OWASP LLM Top 10 / MITRE ATLAS / NIST AI RMF, incl. payment and MCP tool-abuse families whose verdicts come from captured evidence, not LLM opinion. Reports state what was NOT tested. SARIF, CLI, GH Action.", "topics": [], "category": "Red Teaming & Safety", "momentum": 59, "rank": 110 }, { "name": "SakikoMind", "full_name": "ConstantineChenn/SakikoMind", "slug": "constantinechenn-sakikomind", "url": "https://github.com/ConstantineChenn/SakikoMind", "owner": "ConstantineChenn", "owner_avatar": "https://avatars.githubusercontent.com/u/314436326?v=4", "stars": 112, "forks": 0, "open_issues": 0, "language": "Python", "license": "", "pushed_at": "2026-10-31T07:07:48Z", "created_at": "2026-08-07T21:52:01Z", "description": "Enterprise AI customer-service Agent platform featuring hybrid intent recognition, multi-agent orchestration, MCP-powered RAG, dynamic Skills, memory, human handoff, observability and evaluation. 小祥智能客服系统，支持可追溯、可观测与云端部署。", "topics": [], "category": "Observability", "momentum": 59, "rank": 111 }, { "name": "Awesome-LLM-in-Social-Science", "full_name": "ValueByte-AI/Awesome-LLM-in-Social-Science", "slug": "valuebyte-ai-awesome-llm-in-social-science", "url": "https://github.com/ValueByte-AI/Awesome-LLM-in-Social-Science", "owner": "ValueByte-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/151605979?v=4", "stars": 653, "forks": 57, "open_issues": 2, "language": "", "license": "MIT", "pushed_at": "2026-09-24T11:38:53Z", "created_at": "2023-10-15T13:34:33Z", "description": "Awesome papers involving LLMs in Social Science.", "topics": [ "alignment", "economics", "large-language-models", "llm-agent", "llm-evaluation", "llms", "policy", "psychology", "simulation-environment", "social-network", "social-science" ], "category": "Agent Eval", "momentum": 58, "rank": 112 }, { "name": "AIGoat", "full_name": "AISecurityConsortium/AIGoat", "slug": "aisecurityconsortium-aigoat", "url": "https://github.com/AISecurityConsortium/AIGoat", "owner": "AISecurityConsortium", "owner_avatar": "https://avatars.githubusercontent.com/u/229268502?v=4", "stars": 255, "forks": 83, "open_issues": 3, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-18T20:49:39Z", "created_at": "2025-08-29T05:32:04Z", "description": "AIGoat - Open-source AI security playground for hands-on LLM red teaming. Explore prompt injection, RAG poisoning, AI agents, MCP security, and OWASP Top 10 for LLM Applications through practical attack labs, CTF challenges, and progressive defenses all running locally.", "topics": [ "ai-goat", "ai-playground", "ai-security", "ai-security-lab", "ai-vulnerabilities", "aigoat", "ctf-challenges", "llm-attacks", "llm-red-teaming", "llm-security", "owasp-llm", "prompt-injection", "prompt-injection-testing", "red-teaming" ], "category": "Red Teaming & Safety", "momentum": 58, "rank": 113 }, { "name": "Awesome-RAG-Evaluation", "full_name": "YHPeter/Awesome-RAG-Evaluation", "slug": "yhpeter-awesome-rag-evaluation", "url": "https://github.com/YHPeter/Awesome-RAG-Evaluation", "owner": "YHPeter", "owner_avatar": "https://avatars.githubusercontent.com/u/44126839?v=4", "stars": 200, "forks": 21, "open_issues": 5, "language": "TeX", "license": "MIT", "pushed_at": "2026-10-25T04:24:29Z", "created_at": "2024-03-07T19:05:33Z", "description": "The official repository for the paper: Evaluation of Retrieval-Augmented Generation: A Survey.", "topics": [], "category": "RAG Eval", "momentum": 58, "rank": 114 }, { "name": "RouterArena", "full_name": "RouteWorks/RouterArena", "slug": "routeworks-routerarena", "url": "https://github.com/RouteWorks/RouterArena", "owner": "RouteWorks", "owner_avatar": "https://avatars.githubusercontent.com/u/235972115?v=4", "stars": 155, "forks": 64, "open_issues": 37, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-31T22:54:50Z", "created_at": "2025-10-05T18:38:19Z", "description": "RouterArena: An open framework for evaluating LLM routers with standardized datasets, metrics, an automated framework, and a live leaderboard.", "topics": [ "arena", "llm", "llm-router", "llm-routing", "multi-agent", "multi-agent-systems", "router-benchmark", "router-evaluation", "router-leaderboard", "routing" ], "category": "Agent Eval", "momentum": 58, "rank": 115 }, { "name": "llm-gateway-comparison", "full_name": "apimart001a/llm-gateway-comparison", "slug": "apimart001a-llm-gateway-comparison", "url": "https://github.com/apimart001a/llm-gateway-comparison", "owner": "apimart001a", "owner_avatar": "https://avatars.githubusercontent.com/u/330301702?v=4", "stars": 155, "forks": 0, "open_issues": 0, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-09-20T06:58:36Z", "created_at": "2026-09-17T09:27:10Z", "description": "LLM gateway comparison: self-hosted and managed AI API gateways compared on protocol compatibility, billing units, timeout and retry semantics, idempotency, quotas and observability.", "topics": [ "ai-api-gateway", "api-aggregator", "api-gateway", "api-relay", "apimart", "benchmark", "gateway-comparison", "litellm-alternative", "llm-gateway", "openai-compatible" ], "category": "Observability", "momentum": 58, "rank": 116 }, { "name": "riftcoach-agent", "full_name": "123Cx330Yrx/riftcoach-agent", "slug": "123cx330yrx-riftcoach-agent", "url": "https://github.com/123Cx330Yrx/riftcoach-agent", "owner": "123Cx330Yrx", "owner_avatar": "https://avatars.githubusercontent.com/u/213471647?v=4", "stars": 101, "forks": 2, "open_issues": 8, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T16:01:12Z", "created_at": "2026-07-22T03:08:44Z", "description": "A quality-gated League of Legends post-game coaching agent built on deterministic match data, RAG, and LLM evaluation.", "topics": [ "ai-agent", "fastapi", "league-of-legends", "llm", "python", "rag", "riot-api" ], "category": "RAG Eval", "momentum": 58, "rank": 117 }, { "name": "Noria", "full_name": "AhmadHassan-BTed/Noria", "slug": "ahmadhassan-bted-noria", "url": "https://github.com/AhmadHassan-BTed/Noria", "owner": "AhmadHassan-BTed", "owner_avatar": "https://avatars.githubusercontent.com/u/123323526?v=4", "stars": 4643, "forks": 4, "open_issues": 7, "language": "JavaScript", "license": "MIT", "pushed_at": "2026-07-22T12:34:11Z", "created_at": "2026-05-31T21:56:03Z", "description": "An agentic extractor for messaging platforms. Noria receives messages & URLs via chat; scrapes web data to evaluate the content against custom scoring matrices (e.g., jobs, scholarships); and sends a structured summary with a calculated match score directly to the assigned messenger number.", "topics": [ "ai-agent", "automation", "data-extraction", "event-driven-architecture", "gemini-api", "jina-ai", "llm-pipeline", "puppeteer-stealth", "semantic-analysis", "structured-outputs", "web-scraping", "whatsapp-bot" ], "category": "Agent Eval", "momentum": 57, "rank": 118 }, { "name": "Awesome-AI4Med", "full_name": "FreedomIntelligence/Awesome-AI4Med", "slug": "freedomintelligence-awesome-ai4med", "url": "https://github.com/FreedomIntelligence/Awesome-AI4Med", "owner": "FreedomIntelligence", "owner_avatar": "https://avatars.githubusercontent.com/u/127706844?v=4", "stars": 2901, "forks": 497, "open_issues": 2, "language": "", "license": "", "pushed_at": "2026-08-04T14:35:31Z", "created_at": "2019-11-14T07:50:52Z", "description": "A curated list of medical LLMs, multimodal systems, datasets, benchmarks, and more. 🏥", "topics": [ "awesome-lists", "collection", "datasets", "medical", "medical-imaging", "medical-llms", "medical-vlms", "models" ], "category": "Benchmarks", "momentum": 57, "rank": 119 }, { "name": "smallcode", "full_name": "Doorman11991/smallcode", "slug": "doorman11991-smallcode", "url": "https://github.com/Doorman11991/smallcode", "owner": "Doorman11991", "owner_avatar": "https://avatars.githubusercontent.com/u/283874042?v=4", "stars": 2027, "forks": 152, "open_issues": 21, "language": "JavaScript", "license": "MIT", "pushed_at": "2026-08-12T20:43:58Z", "created_at": "2026-05-18T06:19:44Z", "description": "AI coding agent optimized for small LLMs. 87% benchmark with 4B-active model.", "topics": [], "category": "Agent Eval", "momentum": 57, "rank": 120 }, { "name": "awesome-ai-testing", "full_name": "tugkanboz/awesome-ai-testing", "slug": "tugkanboz-awesome-ai-testing", "url": "https://github.com/tugkanboz/awesome-ai-testing", "owner": "tugkanboz", "owner_avatar": "https://avatars.githubusercontent.com/u/17913643?v=4", "stars": 168, "forks": 69, "open_issues": 14, "language": "", "license": "CC0-1.0", "pushed_at": "2026-10-27T20:10:05Z", "created_at": "2026-05-02T11:20:21Z", "description": "A curated list of AI-powered testing tools, frameworks, and resources for QA engineers. From test generation to self-healing automation, MCP-based testing, LLM evaluation, and more.", "topics": [ "ai-test-automation", "ai-testing", "awesome", "awesome-list", "llm-testing", "mcp-testing", "qa", "software-testing", "test-automation", "test-generation" ], "category": "Eval Frameworks", "momentum": 57, "rank": 121 }, { "name": "traccia-py", "full_name": "traccia-ai/traccia-py", "slug": "traccia-ai-traccia-py", "url": "https://github.com/traccia-ai/traccia-py", "owner": "traccia-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/256045371?v=4", "stars": 131, "forks": 26, "open_issues": 7, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-03T12:08:28Z", "created_at": "2026-01-21T16:12:57Z", "description": "OpenTelemetry-native SDK for AI agent observability, tracing, evaluation, debugging, governance, and runtime policy enforcement. Framework-agnostic and built for OpenAI Agents, LangGraph, CrewAI, and LLM applications in production.", "topics": [ "agent", "agent-debugging", "agent-governance", "agent-governance-autonomous-systems", "agent-monitoring", "agent-monitoring-policy-engine", "agent-optimization", "agent-policies", "agentic-ai", "ai", "observability" ], "category": "Observability", "momentum": 57, "rank": 122 }, { "name": "model-message-stream-protocol", "full_name": "Prism-Shadow/model-message-stream-protocol", "slug": "prism-shadow-model-message-stream-protocol", "url": "https://github.com/Prism-Shadow/model-message-stream-protocol", "owner": "Prism-Shadow", "owner_avatar": "https://avatars.githubusercontent.com/u/236323912?v=4", "stars": 124, "forks": 13, "open_issues": 1, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2026-11-03T12:13:53Z", "created_at": "2026-01-15T14:56:45Z", "description": "One interface for 1,000+ LLMs, with zero-code switching and built-in observability. (GPT 6 / Claude 5 / Gemini 4)", "topics": [ "ai", "anthropic", "claude", "claude-code", "deepseek", "gemini", "google", "gpt", "openai" ], "category": "Observability", "momentum": 57, "rank": 123 }, { "name": "generative-ai-project-template", "full_name": "AmineDjeghri/generative-ai-project-template", "slug": "aminedjeghri-generative-ai-project-template", "url": "https://github.com/AmineDjeghri/generative-ai-project-template", "owner": "AmineDjeghri", "owner_avatar": "https://avatars.githubusercontent.com/u/32715913?v=4", "stars": 118, "forks": 11, "open_issues": 3, "language": "Python", "license": "MIT", "pushed_at": "2026-11-03T05:56:03Z", "created_at": "2024-12-05T17:11:09Z", "description": "Template for a new generative ai project using uv, nicegui, fastapi, llms (cloud & local with litellm and ollama, cpu/gpu) and langfuse for llm observability", "topics": [ "azureopenai", "linux", "macos", "mkdocs", "mlops", "openai", "python", "ubuntu" ], "category": "Observability", "momentum": 57, "rank": 124 }, { "name": "llm-bridge", "full_name": "supermemoryai/llm-bridge", "slug": "supermemoryai-llm-bridge", "url": "https://github.com/supermemoryai/llm-bridge", "owner": "supermemoryai", "owner_avatar": "https://avatars.githubusercontent.com/u/171979587?v=4", "stars": 325, "forks": 34, "open_issues": 4, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-10-02T18:37:40Z", "created_at": "2025-07-04T23:16:20Z", "description": "Interoperability between input formats of various LLMs, with observability, error handling, etc. built in.", "topics": [], "category": "Observability", "momentum": 56, "rank": 125 }, { "name": "AgentEval", "full_name": "AgentEvalHQ/AgentEval", "slug": "agentevalhq-agenteval", "url": "https://github.com/AgentEvalHQ/AgentEval", "owner": "AgentEvalHQ", "owner_avatar": "https://avatars.githubusercontent.com/u/263448867?v=4", "stars": 158, "forks": 16, "open_issues": 3, "language": "C#", "license": "MIT", "pushed_at": "2026-10-25T08:40:23Z", "created_at": "2026-01-02T19:52:13Z", "description": "AgentEval is the comprehensive .NET toolkit for AI agent evaluation—tool usage validation, RAG quality metrics, stochastic evaluation, and model comparison—built first for Microsoft Agent Framework (MAF) and Microsoft.Extensions.AI. What RAGAS, PromptFoo and DeepEval do for Python, AgentEval does for .NET", "topics": [ "agent", "agentic", "evals", "evaluations", "framework", "net", "red-teaming", "testing", "workflows" ], "category": "Red Teaming & Safety", "momentum": 56, "rank": 126 }, { "name": "tma1", "full_name": "tma1-ai/tma1", "slug": "tma1-ai-tma1", "url": "https://github.com/tma1-ai/tma1", "owner": "tma1-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/267869330?v=4", "stars": 124, "forks": 16, "open_issues": 14, "language": "Go", "license": "Apache-2.0", "pushed_at": "2026-10-30T14:08:21Z", "created_at": "2026-03-13T07:38:38Z", "description": "Local-first observability your agent reads back. TMA1 records every LLM call, then routes what it sees into the agent's next turn via hooks and MCP.", "topics": [ "agent-loop", "agent-observability", "ai-agents", "claude-code", "codex", "copilot-cli", "genai", "greptimedb", "llm-observability", "llmops", "local-first", "logs", "mcp", "metrics", "observability", "openclaw", "opentelemetry", "self-hosted", "sql", "tracing" ], "category": "Observability", "momentum": 56, "rank": 127 }, { "name": "tribunal", "full_name": "georgeguimaraes/tribunal", "slug": "georgeguimaraes-tribunal", "url": "https://github.com/georgeguimaraes/tribunal", "owner": "georgeguimaraes", "owner_avatar": "https://avatars.githubusercontent.com/u/2929?v=4", "stars": 119, "forks": 3, "open_issues": 1, "language": "Elixir", "license": "MIT", "pushed_at": "2026-10-26T14:37:58Z", "created_at": "2026-01-12T11:42:30Z", "description": "LLM evaluation framework for Elixir: evaluate and test LLM outputs, detect hallucinations, measure response quality", "topics": [ "ai", "elixir", "evaluation", "hallucination-detection", "llm", "llm-evaluation", "testing" ], "category": "Eval Frameworks", "momentum": 55, "rank": 128 }, { "name": "ares", "full_name": "dreadnode/ares", "slug": "dreadnode-ares", "url": "https://github.com/dreadnode/ares", "owner": "dreadnode", "owner_avatar": "https://avatars.githubusercontent.com/u/350839?v=4", "stars": 100, "forks": 20, "open_issues": 40, "language": "Rust", "license": "MIT", "pushed_at": "2026-10-28T00:15:28Z", "created_at": "2025-12-15T17:15:07Z", "description": "Ares is an autonomous security operations platform where LLM-driven red and blue team agents operate against each other on live infrastructure, enabling realistic evaluation of attack and defense.", "topics": [], "category": "Agent Eval", "momentum": 55, "rank": 129 }, { "name": "llm-observability", "full_name": "DataDog/llm-observability", "slug": "datadog-llm-observability", "url": "https://github.com/DataDog/llm-observability", "owner": "DataDog", "owner_avatar": "https://avatars.githubusercontent.com/u/365230?v=4", "stars": 81, "forks": 14, "open_issues": 16, "language": "Jupyter Notebook", "license": "", "pushed_at": "2026-11-03T00:00:45Z", "created_at": "2024-05-22T14:22:39Z", "description": "Learn by example how to instrument Datadog's LLM Observability product", "topics": [], "category": "Observability", "momentum": 55, "rank": 130 }, { "name": "tensorzero", "full_name": "tensorzero/tensorzero", "slug": "tensorzero-tensorzero", "url": "https://github.com/tensorzero/tensorzero", "owner": "tensorzero", "owner_avatar": "https://avatars.githubusercontent.com/u/148420822?v=4", "stars": 11708, "forks": 967, "open_issues": 392, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-06-11T01:48:44Z", "created_at": "2024-07-16T21:00:53Z", "description": "TensorZero is an open-source LLMOps platform that unifies an LLM gateway, observability, evaluation, optimization, and experimentation.", "topics": [ "ai", "ai-engineering", "anthropic", "artificial-intelligence", "deep-learning", "genai", "generative-ai", "gpt", "large-language-models", "llama", "llm", "llmops", "llms", "machine-learning", "ml", "ml-engineering", "mlops", "openai", "python", "rust" ], "category": "Observability", "momentum": 54, "rank": 131 }, { "name": "rag-in-action", "full_name": "huangjia2019/rag-in-action", "slug": "huangjia2019-rag-in-action", "url": "https://github.com/huangjia2019/rag-in-action", "owner": "huangjia2019", "owner_avatar": "https://avatars.githubusercontent.com/u/48795276?v=4", "stars": 856, "forks": 332, "open_issues": 4, "language": "Jupyter Notebook", "license": "", "pushed_at": "2026-08-25T08:04:25Z", "created_at": "2025-03-06T07:42:09Z", "description": "End-to-end RAG system design, evaluation, and optimization. 极客时间RAG训练营，RAG 10大组件全面拆解，4个实操项目吃透 RAG 全流程。RAG的落地，往往是面向业务做RAG，而不是反过来面向RAG做业务。这就是为什么我们需要针对不同场景、不同问题做针对性的调整、优化和定制化。魔鬼全在细节中，我们深入进去探究。", "topics": [ "agent", "ai", "llm", "rag" ], "category": "RAG Eval", "momentum": 54, "rank": 132 }, { "name": "ClawProBench", "full_name": "suyoumo/ClawProBench", "slug": "suyoumo-clawprobench", "url": "https://github.com/suyoumo/ClawProBench", "owner": "suyoumo", "owner_avatar": "https://avatars.githubusercontent.com/u/39437291?v=4", "stars": 829, "forks": 54, "open_issues": 0, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-08-25T11:47:44Z", "created_at": "2025-03-02T09:15:10Z", "description": "ClawProBench is a live-first benchmark harness for evaluating LLM agents in the OpenClaw runtime with deterministic grading and repeated-trial reliability.", "topics": [ "agent", "benchmark", "evaluation", "harness", "leaderboard", "llm", "openclaw" ], "category": "Agent Eval", "momentum": 54, "rank": 133 }, { "name": "EnterpriseRAG-Bench", "full_name": "onyx-dot-app/EnterpriseRAG-Bench", "slug": "onyx-dot-app-enterpriserag-bench", "url": "https://github.com/onyx-dot-app/EnterpriseRAG-Bench", "owner": "onyx-dot-app", "owner_avatar": "https://avatars.githubusercontent.com/u/131946000?v=4", "stars": 611, "forks": 72, "open_issues": 13, "language": "", "license": "MIT", "pushed_at": "2026-09-03T23:20:07Z", "created_at": "2026-02-19T02:48:25Z", "description": "Dataset and benchmark for RAG on company internal documents.", "topics": [ "benchmark", "dataset", "enterprise", "enterprise-search", "evaluation", "generative-ai", "information-retrieval", "knowledge-base", "large-language-models", "llm", "llm-evaluation", "python", "question-answering", "rag", "rag-evaluation", "retrieval-augmented-generation", "semantic-search", "synthetic-data", "synthetic-data-generation" ], "category": "RAG Eval", "momentum": 54, "rank": 134 }, { "name": "FinSight-AI", "full_name": "juanjuandog/FinSight-AI", "slug": "juanjuandog-finsight-ai", "url": "https://github.com/juanjuandog/FinSight-AI", "owner": "juanjuandog", "owner_avatar": "https://avatars.githubusercontent.com/u/212179655?v=4", "stars": 558, "forks": 32, "open_issues": 12, "language": "Java", "license": "MIT", "pushed_at": "2026-09-02T13:54:32Z", "created_at": "2026-05-11T11:19:40Z", "description": "AI equity research agent with resilient workflows, evidence-grounded RAG, versioned reports, and automated quality evaluation.", "topics": [ "ai-agent", "financial-research", "llm-evaluation", "pgvector", "postgresql", "rabbitmq", "rag", "redis", "spring-boot", "workflow-orchestration" ], "category": "RAG Eval", "momentum": 54, "rank": 135 }, { "name": "Awesome-Offensive-AI-Agentic-Landscape", "full_name": "Yeti-791/Awesome-Offensive-AI-Agentic-Landscape", "slug": "yeti-791-awesome-offensive-ai-agentic-landscape", "url": "https://github.com/Yeti-791/Awesome-Offensive-AI-Agentic-Landscape", "owner": "Yeti-791", "owner_avatar": "https://avatars.githubusercontent.com/u/12783061?v=4", "stars": 422, "forks": 51, "open_issues": 0, "language": "", "license": "MIT", "pushed_at": "2026-09-10T17:05:28Z", "created_at": "2026-06-11T03:32:26Z", "description": "This document curates open-source projects, academic papers, capability benchmarks, and commercial solutions (international & China) in AI penetration testing, LLM red teaming, autonomous offensive agents, and vulnerability discovery—aimed at helping researchers, security engineers, and enterprise decision-makers quickly form a holistic view.", "topics": [ "ai-benchmarks", "ai-hacking", "ai-pentesting", "ai-red-teaming", "ai-security", "autonomous-penetration", "awesome-list", "ctf", "cybersecurity", "hacking-agents", "hacking-tool", "intelligent-penetration", "llm-security", "offensive-ai", "offensive-security", "pentest", "pentest-tool", "redteam", "security-tools" ], "category": "Red Teaming & Safety", "momentum": 54, "rank": 136 }, { "name": "BinaryAudit", "full_name": "QuesmaOrg/BinaryAudit", "slug": "quesmaorg-binaryaudit", "url": "https://github.com/QuesmaOrg/BinaryAudit", "owner": "QuesmaOrg", "owner_avatar": "https://avatars.githubusercontent.com/u/152969654?v=4", "stars": 107, "forks": 7, "open_issues": 3, "language": "Shell", "license": "", "pushed_at": "2026-10-22T14:27:49Z", "created_at": "2025-12-11T16:19:19Z", "description": "An open-source benchmark for evaluating AI agents' ability to find backdoors hidden in compiled binaries.", "topics": [ "ai", "benchmark", "binary-analysis", "cybersecurity", "llm-eval", "reverse-engineering" ], "category": "Agent Eval", "momentum": 54, "rank": 137 }, { "name": "promptfoo-action", "full_name": "promptfoo/promptfoo-action", "slug": "promptfoo-promptfoo-action", "url": "https://github.com/promptfoo/promptfoo-action", "owner": "promptfoo", "owner_avatar": "https://avatars.githubusercontent.com/u/137907881?v=4", "stars": 80, "forks": 36, "open_issues": 21, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-11-02T17:08:51Z", "created_at": "2023-06-21T03:10:38Z", "description": "The GitHub Action for Promptfoo. Test your prompts, agents, and RAGs. AI Red teaming, pentesting, and vulnerability scanning for LLMs. Compare performance of GPT, Claude, Gemini, Llama, and more. Simple declarative configs with command line and CI/CD integration.", "topics": [ "actions", "llm", "prompt-engineering", "rag", "security" ], "category": "Red Teaming & Safety", "momentum": 54, "rank": 138 }, { "name": "pojia-breaker", "full_name": "Justinuse1/pojia-breaker", "slug": "justinuse1-pojia-breaker", "url": "https://github.com/Justinuse1/pojia-breaker", "owner": "Justinuse1", "owner_avatar": "https://avatars.githubusercontent.com/u/122102111?v=4", "stars": 49, "forks": 12, "open_issues": 2, "language": "JavaScript", "license": "MIT", "pushed_at": "2026-09-27T07:15:24Z", "created_at": "2026-09-24T03:17:53Z", "description": "Chinese-first LLM jailbreak benchmark & red-team toolkit | 中文LLM安全评测框架 | by POJIA.AI", "topics": [ "ai-safety", "benchmark", "chinese-nlp", "deepseek", "jailbreak", "llm-evaluation", "llm-security", "prompt-injection", "red-team", "security-research" ], "category": "Red Teaming & Safety", "momentum": 54, "rank": 139 }, { "name": "dsh-design-qa", "full_name": "sunxin-ai/dsh-design-qa", "slug": "sunxin-ai-dsh-design-qa", "url": "https://github.com/sunxin-ai/dsh-design-qa", "owner": "sunxin-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/316078500?v=4", "stars": 44, "forks": 3, "open_issues": 0, "language": "JavaScript", "license": "MIT", "pushed_at": "2026-10-20T12:25:14Z", "created_at": "2026-08-17T14:33:44Z", "description": "Design-fidelity QA for DeepSeek Harness: lend any text-only model an eye, then judge whether the implementation matches the mock. Ships the benchmark behind that judgement — four fixtures, 23 injected defects, and every raw model transcript. Retires itself when DeepSeek ships vision.", "topics": [ "benchmark", "deepseek-harness", "design-qa", "design-review", "dsh", "dsh-plugin", "llm-eval", "multimodal", "openai-compatible", "qwen-vl", "vision", "visual-regression" ], "category": "Benchmarks", "momentum": 54, "rank": 140 }, { "name": "agentops", "full_name": "AgentOps-AI/agentops", "slug": "agentops-ai-agentops", "url": "https://github.com/AgentOps-AI/agentops", "owner": "AgentOps-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/140554352?v=4", "stars": 5940, "forks": 656, "open_issues": 200, "language": "Python", "license": "MIT", "pushed_at": "2026-06-25T08:25:03Z", "created_at": "2023-08-15T23:26:23Z", "description": "Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI", "topics": [ "agent", "agentops", "agents-sdk", "ai", "anthropic", "autogen", "cost-estimation", "crewai", "evals", "evaluation-metrics", "groq", "langchain", "llm", "mistral", "ollama", "openai", "openai-agents" ], "category": "Observability", "momentum": 53, "rank": 141 }, { "name": "Awesome-LLM-based-Text2SQL", "full_name": "DEEP-PolyU/Awesome-LLM-based-Text2SQL", "slug": "deep-polyu-awesome-llm-based-text2sql", "url": "https://github.com/DEEP-PolyU/Awesome-LLM-based-Text2SQL", "owner": "DEEP-PolyU", "owner_avatar": "https://avatars.githubusercontent.com/u/76692159?v=4", "stars": 1378, "forks": 130, "open_issues": 2, "language": "", "license": "MIT", "pushed_at": "2026-08-03T00:50:21Z", "created_at": "2025-09-13T16:30:57Z", "description": "[TKDE2025] Next-Generation Database Interfaces: A Survey of LLM-based Text-to-SQL | A curated list of resources (surveys, papers, benchmarks, and opensource projects) on large language model-based text-to-SQL.", "topics": [ "awesome", "awesome-text-to-sql", "awesome-text2sql", "database", "large-language-models", "llms", "natual-language-processing", "natural-language-understanding", "nl2sql", "survey", "text-to-sql", "text2sql" ], "category": "Benchmarks", "momentum": 53, "rank": 142 }, { "name": "apo", "full_name": "CloudDetail/apo", "slug": "clouddetail-apo", "url": "https://github.com/CloudDetail/apo", "owner": "CloudDetail", "owner_avatar": "https://avatars.githubusercontent.com/u/177638107?v=4", "stars": 374, "forks": 53, "open_issues": 2, "language": "Go", "license": "Apache-2.0", "pushed_at": "2026-09-11T06:30:31Z", "created_at": "2024-08-06T11:43:15Z", "description": "APO is a comprehensive observability platform combining OpenTelemetry with eBPF. Leveraging LLM to enable automated analysis and troubleshooting 🚀.", "topics": [ "apm", "ebpf", "logs", "metrics", "microservice", "monitoring", "observability", "opentelemetry", "tracing" ], "category": "Observability", "momentum": 53, "rank": 143 }, { "name": "thinkingbox", "full_name": "microsoft/thinkingbox", "slug": "microsoft-thinkingbox", "url": "https://github.com/microsoft/thinkingbox", "owner": "microsoft", "owner_avatar": "https://avatars.githubusercontent.com/u/6154722?v=4", "stars": 112, "forks": 13, "open_issues": 16, "language": "Python", "license": "MIT", "pushed_at": "2026-10-15T20:22:28Z", "created_at": "2026-04-22T19:01:10Z", "description": "thinkingbox is a framework for defining tool as MCP servers, running LLM agents against them, and evaluating agent behavior — for offline training-data generation, reinforcement-learning training loops, and model evaluation.", "topics": [], "category": "Agent Eval", "momentum": 53, "rank": 144 }, { "name": "nuguard", "full_name": "NuGuardAI/nuguard", "slug": "nuguardai-nuguard", "url": "https://github.com/NuGuardAI/nuguard", "owner": "NuGuardAI", "owner_avatar": "https://avatars.githubusercontent.com/u/227518070?v=4", "stars": 68, "forks": 22, "open_issues": 35, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-11-01T02:47:55Z", "created_at": "2026-03-20T01:51:54Z", "description": "AI red-teaming tool and LLM security framework to evaluate agentic AI applications. Tests prompt injections, handles vulnerability assessment, SBOM generation, and static analysis.", "topics": [ "adversarial-machine-learning", "agentic-ai", "ai-agents", "ai-red-team-tool", "ai-red-teaming", "ai-security", "ai-tools", "generative-ai", "llm-security", "mcp", "mcp-server", "owasp", "pentest", "prompt-injections", "sbom", "sbom-generator", "static-analysis", "vulnerability-assessment" ], "category": "Red Teaming & Safety", "momentum": 53, "rank": 145 }, { "name": "all-agentic-architectures", "full_name": "FareedKhan-dev/all-agentic-architectures", "slug": "fareedkhan-dev-all-agentic-architectures", "url": "https://github.com/FareedKhan-dev/all-agentic-architectures", "owner": "FareedKhan-dev", "owner_avatar": "https://avatars.githubusercontent.com/u/63067900?v=4", "stars": 4655, "forks": 786, "open_issues": 13, "language": "Jupyter Notebook", "license": "MIT", "pushed_at": "2026-06-22T06:16:53Z", "created_at": "2025-09-24T11:47:58Z", "description": "35 production-grade agentic AI architectures (Reflexion, LATS, GraphRAG, MemGPT, Voyager, BrowserAgent, ...) — a Python library and runnable textbook with multi-provider LLM support and a 17-task benchmark leaderboard.", "topics": [ "agentic-ai", "ai-agents", "langchain", "langgraph", "langsmith", "llm" ], "category": "Agent Eval", "momentum": 52, "rank": 146 }, { "name": "terminal-bench-1", "full_name": "harbor-framework/terminal-bench-1", "slug": "harbor-framework-terminal-bench-1", "url": "https://github.com/harbor-framework/terminal-bench-1", "owner": "harbor-framework", "owner_avatar": "https://avatars.githubusercontent.com/u/256467836?v=4", "stars": 2614, "forks": 568, "open_issues": 319, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-07-11T05:19:08Z", "created_at": "2025-01-17T22:34:26Z", "description": "A benchmark for LLMs on complicated tasks in the terminal", "topics": [], "category": "Benchmarks", "momentum": 52, "rank": 147 }, { "name": "hermes-katana", "full_name": "claudlos/hermes-katana", "slug": "claudlos-hermes-katana", "url": "https://github.com/claudlos/hermes-katana", "owner": "claudlos", "owner_avatar": "https://avatars.githubusercontent.com/u/258678933?v=4", "stars": 49, "forks": 3, "open_issues": 6, "language": "Python", "license": "MIT", "pushed_at": "2026-11-02T12:15:46Z", "created_at": "2026-04-05T02:57:13Z", "description": "Defense-in-depth security toolkit for LLM agents — taint tracking, proxy secret guard, policy engine, and red-team benchmarking", "topics": [ "agent-security", "ai", "ai-agent", "ai-agents", "ai-security", "hermes", "hermes-agent", "llm", "llm-security", "llms", "mitm-proxy", "prompt-injection", "python", "security", "security-tools", "taint-tracking" ], "category": "Red Teaming & Safety", "momentum": 52, "rank": 148 }, { "name": "ragrank", "full_name": "izam-mohammed/ragrank", "slug": "izam-mohammed-ragrank", "url": "https://github.com/izam-mohammed/ragrank", "owner": "izam-mohammed", "owner_avatar": "https://avatars.githubusercontent.com/u/106471909?v=4", "stars": 47, "forks": 15, "open_issues": 1, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-11-02T14:49:37Z", "created_at": "2024-02-17T18:32:08Z", "description": "🎯 Your free LLM evaluation toolkit helps you assess the accuracy of facts, how well it understands context, its tone, and more. This helps you see how good your LLM applications are.", "topics": [ "evaluation", "language-model", "llm", "llm-eval", "llmops", "machine-learning", "prompt-engineering", "python", "rag" ], "category": "RAG Eval", "momentum": 52, "rank": 149 }, { "name": "evals", "full_name": "openai/evals", "slug": "openai-evals", "url": "https://github.com/openai/evals", "owner": "openai", "owner_avatar": "https://avatars.githubusercontent.com/u/14957082?v=4", "stars": 19691, "forks": 3128, "open_issues": 351, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-04-14T15:29:57Z", "created_at": "2023-01-23T20:51:04Z", "description": "Evals is a framework for evaluating LLMs and LLM systems, and an open-source registry of benchmarks.", "topics": [], "category": "Benchmarks", "momentum": 51, "rank": 150 }, { "name": "ai_for_the_win", "full_name": "depalmar/ai_for_the_win", "slug": "depalmar-ai-for-the-win", "url": "https://github.com/depalmar/ai_for_the_win", "owner": "depalmar", "owner_avatar": "https://avatars.githubusercontent.com/u/57051011?v=4", "stars": 170, "forks": 25, "open_issues": 9, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-09-25T07:04:33Z", "created_at": "2025-12-15T23:55:09Z", "description": "Build AI-powered security tools. 50+ hands-on labs covering ML, LLMs, RAG, threat detection, DFIR, and red teaming. Includes Colab notebooks, Docker environment, and CTF challenges.", "topics": [ "adversarial-ml", "ai", "blue-team", "cloud-security", "ctf", "cybersecurity", "dfir", "docker", "hands-on-labs", "incident-response", "llm", "machine-learning", "malware-analysis", "python", "security-training", "siem", "threat-detection", "threat-hunting", "threat-intelligence", "xdr" ], "category": "Red Teaming & Safety", "momentum": 51, "rank": 151 }, { "name": "Jailbreaker-CE", "full_name": "SpecterOps/Jailbreaker-CE", "slug": "specterops-jailbreaker-ce", "url": "https://github.com/SpecterOps/Jailbreaker-CE", "owner": "SpecterOps", "owner_avatar": "https://avatars.githubusercontent.com/u/25406560?v=4", "stars": 115, "forks": 7, "open_issues": 7, "language": "Python", "license": "BSD-3-Clause", "pushed_at": "2026-10-02T05:55:39Z", "created_at": "2026-06-26T15:46:36Z", "description": "Jailbreaker is a local evaluation tool for testing chatbot and agent-style systems against jailbreak, prompt-injection, and related failure modes.", "topics": [], "category": "Red Teaming & Safety", "momentum": 51, "rank": 152 }, { "name": "redthread", "full_name": "matheusht/redthread", "slug": "matheusht-redthread", "url": "https://github.com/matheusht/redthread", "owner": "matheusht", "owner_avatar": "https://avatars.githubusercontent.com/u/96798246?v=4", "stars": 53, "forks": 6, "open_issues": 22, "language": "Python", "license": "MIT", "pushed_at": "2026-10-23T17:57:53Z", "created_at": "2026-03-31T17:20:12Z", "description": "An autonomous red-teaming engine for LLMs. RedThread manages the full security lifecycle: generating adversarial attacks, executing precision evaluations, and synthesizing validated guardrails for safe self-improvement.", "topics": [], "category": "Red Teaming & Safety", "momentum": 51, "rank": 153 }, { "name": "crucible", "full_name": "crucible-security/crucible", "slug": "crucible-security-crucible", "url": "https://github.com/crucible-security/crucible", "owner": "crucible-security", "owner_avatar": "https://avatars.githubusercontent.com/u/277894632?v=4", "stars": 51, "forks": 32, "open_issues": 21, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-10-25T11:19:00Z", "created_at": "2026-04-20T21:54:13Z", "description": "pytest for AI agents - Autonomous red-teaming, behavioral monitoring & security testing for LLM agents", "topics": [ "agentic-ai", "ai-agents", "ai-security", "edulinkup", "elusoc", "hacktoberfest", "llm", "llm-security", "owasp", "prompt-injection", "python", "red-team", "red-teaming", "security-testing" ], "category": "Observability", "momentum": 51, "rank": 154 }, { "name": "aludel", "full_name": "cristianodabc/aludel", "slug": "cristianodabc-aludel", "url": "https://github.com/cristianodabc/aludel", "owner": "cristianodabc", "owner_avatar": "https://avatars.githubusercontent.com/u/14858867?v=4", "stars": 41, "forks": 1, "open_issues": 6, "language": "Elixir", "license": "Apache-2.0", "pushed_at": "2026-11-02T09:27:25Z", "created_at": "2026-03-20T01:15:56Z", "description": "Phoenix-native LLM evaluation, prompt testing, model comparison, red teaming, and observability for Elixir", "topics": [ "ai", "ai-testing", "elixir", "elixir-library", "evals", "exunit", "hex-package", "live-view", "llm", "llm-evaluation", "llm-observability", "llmops", "model-comparison", "phoenix", "phoenix-liveview", "prompt-engineering", "prompt-testing", "red-teaming", "workbench" ], "category": "Observability", "momentum": 51, "rank": 155 }, { "name": "ragas", "full_name": "vibrantlabsai/ragas", "slug": "vibrantlabsai-ragas", "url": "https://github.com/vibrantlabsai/ragas", "owner": "vibrantlabsai", "owner_avatar": "https://avatars.githubusercontent.com/u/122604797?v=4", "stars": 16224, "forks": 1798, "open_issues": 649, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-02-24T07:47:19Z", "created_at": "2023-05-08T17:48:04Z", "description": "Supercharge Your LLM Application Evaluations 🚀", "topics": [ "evaluation", "llm", "llmops" ], "category": "Eval Frameworks", "momentum": 50, "rank": 156 }, { "name": "RagaAI-Catalyst", "full_name": "raga-ai-hub/RagaAI-Catalyst", "slug": "raga-ai-hub-ragaai-catalyst", "url": "https://github.com/raga-ai-hub/RagaAI-Catalyst", "owner": "raga-ai-hub", "owner_avatar": "https://avatars.githubusercontent.com/u/161833182?v=4", "stars": 16188, "forks": 3575, "open_issues": 35, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-02-11T14:43:33Z", "created_at": "2024-08-26T12:13:15Z", "description": "Python SDK for Agent AI Observability, Monitoring and Evaluation Framework. Includes features like agent, llm and tools tracing, debugging multi-agentic system, self-hosted dashboard and advanced analytics with timeline and execution graph view", "topics": [ "agentic-ai", "agentic-ai-development", "agentneo", "agents", "ai-agent-monitoring", "ai-application-debugging", "ai-evaluation-tools", "ai-performance-optimization", "ai-tool-interaction-monitoring", "llm-testing", "llm-tracing", "llmops" ], "category": "Observability", "momentum": 50, "rank": 157 }, { "name": "leva", "full_name": "kieranklaassen/leva", "slug": "kieranklaassen-leva", "url": "https://github.com/kieranklaassen/leva", "owner": "kieranklaassen", "owner_avatar": "https://avatars.githubusercontent.com/u/209089?v=4", "stars": 147, "forks": 9, "open_issues": 6, "language": "Ruby", "license": "MIT", "pushed_at": "2026-09-13T01:19:33Z", "created_at": "2024-08-13T17:41:18Z", "description": "LLM Evaluation Framework for Rails apps to be used with production data.", "topics": [ "llm", "llm-evaluation", "ruby-on-rails" ], "category": "Eval Frameworks", "momentum": 49, "rank": 158 }, { "name": "basilisk", "full_name": "regaan/basilisk", "slug": "regaan-basilisk", "url": "https://github.com/regaan/basilisk", "owner": "regaan", "owner_avatar": "https://avatars.githubusercontent.com/u/78273014?v=4", "stars": 84, "forks": 12, "open_issues": 0, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-10-01T15:51:31Z", "created_at": "2026-03-03T11:33:37Z", "description": "Basilisk — Open-source AI red teaming framework with genetic prompt evolution. Automated LLM security testing for GPT-4, Claude, Grok, Gemini. OWASP LLM Top 10 coverage. 32 attack modules.", "topics": [ "adversarial-ai", "ai", "ai-red-teaming", "ai-security", "basilisk", "chatgpt-security", "genetic-algorithm", "jailbreak", "llm", "llm-security", "llm-security-compliance-prompt-injection", "offensive-security", "owasp", "owasp-llm-top-10", "pentesting", "prompt-injection", "red-team", "red-teaming", "security", "vulnerability-scanner" ], "category": "Red Teaming & Safety", "momentum": 49, "rank": 159 }, { "name": "valor", "full_name": "Striveworks/valor", "slug": "striveworks-valor", "url": "https://github.com/Striveworks/valor", "owner": "Striveworks", "owner_avatar": "https://avatars.githubusercontent.com/u/45523576?v=4", "stars": 40, "forks": 4, "open_issues": 21, "language": "Python", "license": "MIT", "pushed_at": "2026-10-21T19:34:19Z", "created_at": "2022-12-27T14:27:17Z", "description": "Valor is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models.", "topics": [ "classification", "computer-vision", "evaluation", "evaluation-metrics", "image-segmentation", "llm-eval", "mlops", "model-evaluation", "nlp", "object-detection", "text-generation" ], "category": "Eval Frameworks", "momentum": 49, "rank": 160 }, { "name": "industrial-general-rag", "full_name": "phoenix-zhou/industrial-general-rag", "slug": "phoenix-zhou-industrial-general-rag", "url": "https://github.com/phoenix-zhou/industrial-general-rag", "owner": "phoenix-zhou", "owner_avatar": "https://avatars.githubusercontent.com/u/15656570?v=4", "stars": 619, "forks": 30, "open_issues": 0, "language": "Python", "license": "", "pushed_at": "2026-07-22T01:10:05Z", "created_at": "2026-07-21T13:59:07Z", "description": "The project integrates the RAG and FAQ dual mechanisms, introduces intent recognition, dynamic adaptive retrieval, and the RAGAS automated evaluation system, achieving a transformation from empirical optimization to data-driven optimization.", "topics": [], "category": "RAG Eval", "momentum": 48, "rank": 161 }, { "name": "Eval", "full_name": "ai-twinkle/Eval", "slug": "ai-twinkle-eval", "url": "https://github.com/ai-twinkle/Eval", "owner": "ai-twinkle", "owner_avatar": "https://avatars.githubusercontent.com/u/204453842?v=4", "stars": 117, "forks": 17, "open_issues": 31, "language": "Python", "license": "MIT", "pushed_at": "2026-09-15T15:14:07Z", "created_at": "2025-03-31T07:16:11Z", "description": "High-performance LLM evaluation framework with parallel API calls — up to 17× faster than sequential tools. Supports box, math, and logit-based evaluation.", "topics": [ "eval", "evaluation", "llm" ], "category": "Reasoning", "momentum": 48, "rank": 162 }, { "name": "skillcheck", "full_name": "sx4im/skillcheck", "slug": "sx4im-skillcheck", "url": "https://github.com/sx4im/skillcheck", "owner": "sx4im", "owner_avatar": "https://avatars.githubusercontent.com/u/128062316?v=4", "stars": 29, "forks": 6, "open_issues": 8, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-10-27T16:44:40Z", "created_at": "2026-06-04T14:23:07Z", "description": "Blind A/B testing for AI agent skills, .cursorrules, CLAUDE.md, and system prompts. Detects placebo and harmful instructions.", "topics": [ "ab-testing", "agent-skills", "agentic-ai", "ai-agents", "benchmark", "claude-code", "cli", "copilot-instructions", "cursor-rules", "cursorrules", "developer-tools", "evals", "llm", "llm-eval", "prompt-engineering", "prompt-evaluation", "prompt-testing", "system-prompts", "typescript", "windsurf" ], "category": "Agent Eval", "momentum": 48, "rank": 163 }, { "name": "skill", "full_name": "pinchbench/skill", "slug": "pinchbench-skill", "url": "https://github.com/pinchbench/skill", "owner": "pinchbench", "owner_avatar": "https://avatars.githubusercontent.com/u/260781745?v=4", "stars": 1375, "forks": 159, "open_issues": 26, "language": "Python", "license": "MIT", "pushed_at": "2026-07-02T01:44:55Z", "created_at": "2026-02-11T01:41:39Z", "description": "PinchBench is a benchmarking system for evaluating LLM models as OpenClaw coding agents. Made with 🦀 by the humans at https://kilo.ai", "topics": [], "category": "Agent Eval", "momentum": 47, "rank": 164 }, { "name": "MMMU", "full_name": "MMMU-Benchmark/MMMU", "slug": "mmmu-benchmark-mmmu", "url": "https://github.com/MMMU-Benchmark/MMMU", "owner": "MMMU-Benchmark", "owner_avatar": "https://avatars.githubusercontent.com/u/151861048?v=4", "stars": 600, "forks": 57, "open_issues": 50, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-07-28T02:29:26Z", "created_at": "2023-11-23T15:53:31Z", "description": "This repo contains evaluation code for the paper \"MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI\"", "topics": [ "computer-vision", "deep-learning", "deep-neural-networks", "evaluation", "foundation-models", "large-language-models", "large-multimodal-models", "llm", "llms", "machine-learning", "multimodal", "multimodal-deep-learning", "multimodal-learning", "multimodality", "natural-language-processing", "question-answering", "stem", "visual-question-answering" ], "category": "Coding Eval", "momentum": 47, "rank": 165 }, { "name": "Intents", "full_name": "coryparrry/Intents", "slug": "coryparrry-intents", "url": "https://github.com/coryparrry/Intents", "owner": "coryparrry", "owner_avatar": "https://avatars.githubusercontent.com/u/222280632?v=4", "stars": 18, "forks": 0, "open_issues": 2, "language": "Swift", "license": "MIT", "pushed_at": "2026-11-02T02:05:30Z", "created_at": "2026-09-05T20:45:13Z", "description": "Intents is a native macOS workbench for evaluating Apple's on-device Foundation Models.", "topics": [ "ai-agents", "apple", "apple-foundation-models", "apple-intelligence", "foundation-evals", "goldengate", "llm-eval", "llm-evaluation", "macos", "macos27", "mcp", "mcp-server", "model-context-protocol", "on-device-ai", "swift", "swiftui" ], "category": "Agent Eval", "momentum": 47, "rank": 166 }, { "name": "EmoLLM", "full_name": "SmartFlowAI/EmoLLM", "slug": "smartflowai-emollm", "url": "https://github.com/SmartFlowAI/EmoLLM", "owner": "SmartFlowAI", "owner_avatar": "https://avatars.githubusercontent.com/u/161115059?v=4", "stars": 1798, "forks": 225, "open_issues": 6, "language": "Python", "license": "MIT", "pushed_at": "2026-06-18T16:47:43Z", "created_at": "2024-01-11T08:58:19Z", "description": "心理健康大模型 (LLM x Mental Health), Pre & Post-training & Dataset & Evaluation & Depoly & RAG, with InternLM / Qwen / Baichuan / DeepSeek / Mixtral / LLama / GLM series models", "topics": [ "dataset", "depoly", "evaluation", "llm", "post-training", "the-big-model-of-mental-health" ], "category": "RAG Eval", "momentum": 46, "rank": 167 }, { "name": "QwenFeat-Vocal-Score", "full_name": "CarlWangChina/QwenFeat-Vocal-Score", "slug": "carlwangchina-qwenfeat-vocal-score", "url": "https://github.com/CarlWangChina/QwenFeat-Vocal-Score", "owner": "CarlWangChina", "owner_avatar": "https://avatars.githubusercontent.com/u/41322045?v=4", "stars": 201, "forks": 6, "open_issues": 0, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-08-19T15:44:18Z", "created_at": "2025-07-04T07:22:20Z", "description": "VocalVerse: A powerful vocal evaluation framework powered by the Qwen LLMs", "topics": [], "category": "Eval Frameworks", "momentum": 46, "rank": 168 }, { "name": "LLM-Engineers-Handbook", "full_name": "PacktPublishing/LLM-Engineers-Handbook", "slug": "packtpublishing-llm-engineers-handbook", "url": "https://github.com/PacktPublishing/LLM-Engineers-Handbook", "owner": "PacktPublishing", "owner_avatar": "https://avatars.githubusercontent.com/u/10974906?v=4", "stars": 5415, "forks": 1330, "open_issues": 36, "language": "Python", "license": "MIT", "pushed_at": "2026-04-22T08:25:03Z", "created_at": "2024-04-09T10:56:03Z", "description": "The LLM's practical guide: From the fundamentals to deploying advanced LLM and RAG apps to AWS using LLMOps best practices", "topics": [ "aws", "fine-tuning-llm", "genai", "llm", "llm-evaluation", "llmops", "ml-system-design", "mlops", "rag" ], "category": "RAG Eval", "momentum": 44, "rank": 169 }, { "name": "Rubric4Setwise", "full_name": "Rubric4Setwise/Rubric4Setwise", "slug": "rubric4setwise-rubric4setwise", "url": "https://github.com/Rubric4Setwise/Rubric4Setwise", "owner": "Rubric4Setwise", "owner_avatar": "https://avatars.githubusercontent.com/u/307132082?v=4", "stars": 192, "forks": 1, "open_issues": 1, "language": "Python", "license": "", "pushed_at": "2026-07-30T08:26:45Z", "created_at": "2026-07-21T12:57:04Z", "description": "Rubric-oriented document set evaluation and selection for RAG.", "topics": [], "category": "RAG Eval", "momentum": 44, "rank": 170 }, { "name": "modelpointer", "full_name": "modelpointer/modelpointer", "slug": "modelpointer-modelpointer", "url": "https://github.com/modelpointer/modelpointer", "owner": "modelpointer", "owner_avatar": "https://avatars.githubusercontent.com/u/292158925?v=4", "stars": 115, "forks": 2, "open_issues": 0, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-08-24T14:44:22Z", "created_at": "2026-06-28T08:50:34Z", "description": "A high-performance AI gateway written in Rust. It sits in front of your LLM backends and exposes a unified API surface to clients, handling routing, load balancing, rate limiting, and observability.", "topics": [], "category": "Observability", "momentum": 44, "rank": 171 }, { "name": "PaperGuru-Benchmark", "full_name": "AutoTrustAI/PaperGuru-Benchmark", "slug": "autotrustai-paperguru-benchmark", "url": "https://github.com/AutoTrustAI/PaperGuru-Benchmark", "owner": "AutoTrustAI", "owner_avatar": "https://avatars.githubusercontent.com/u/280205353?v=4", "stars": 1329, "forks": 197, "open_issues": 0, "language": "TeX", "license": "NOASSERTION", "pushed_at": "2026-06-08T21:14:07Z", "created_at": "2026-05-08T16:01:28Z", "description": "Lifecycle-Aware Memory for long-horizon LLM agents — 66.05% on PaperBench, 94.66% on SurveyBench, 10 peer-reviewed acceptances at FSE/ICML/TOSEM/AEI/ICoGB", "topics": [], "category": "Agent Eval", "momentum": 43, "rank": 172 }, { "name": "nasde-toolkit", "full_name": "NoesisVision/nasde-toolkit", "slug": "noesisvision-nasde-toolkit", "url": "https://github.com/NoesisVision/nasde-toolkit", "owner": "NoesisVision", "owner_avatar": "https://avatars.githubusercontent.com/u/128144584?v=4", "stars": 14, "forks": 0, "open_issues": 1, "language": "Python", "license": "MIT", "pushed_at": "2026-10-15T15:33:57Z", "created_at": "2026-03-13T20:01:06Z", "description": "CLI for benchmarks & evals of AI coding agents — on tasks you already understand, using your Claude / Codex / Gemini individual subscriptions or API keys.", "topics": [ "agent-benchmark", "agent-evaluation", "ai-coding-agents", "ai-evaluation", "claude-code", "claude-skills", "cli-tool", "codex", "evals", "gemini-cli", "harbor", "llm-as-a-judge", "llm-eval", "llm-evaluation", "mcp", "model-context-protocol", "opik", "python", "sandbox", "sdlc" ], "category": "Agent Eval", "momentum": 43, "rank": 173 }, { "name": "AgentBench", "full_name": "THUDM/AgentBench", "slug": "thudm-agentbench", "url": "https://github.com/THUDM/AgentBench", "owner": "THUDM", "owner_avatar": "https://avatars.githubusercontent.com/u/48590610?v=4", "stars": 3808, "forks": 294, "open_issues": 80, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-02-08T17:01:05Z", "created_at": "2023-07-28T04:32:06Z", "description": "A Comprehensive Benchmark to Evaluate LLMs as Agents (ICLR'24)", "topics": [ "chatgpt", "gpt-4", "llm", "llm-agent" ], "category": "Agent Eval", "momentum": 42, "rank": 174 }, { "name": "llm-systems-engineering-roadmap", "full_name": "h9-tec/llm-systems-engineering-roadmap", "slug": "h9-tec-llm-systems-engineering-roadmap", "url": "https://github.com/h9-tec/llm-systems-engineering-roadmap", "owner": "h9-tec", "owner_avatar": "https://avatars.githubusercontent.com/u/61054142?v=4", "stars": 206, "forks": 28, "open_issues": 0, "language": "", "license": "", "pushed_at": "2026-07-27T09:47:21Z", "created_at": "2026-04-25T13:18:12Z", "description": "A practical roadmap for mastering LLM internals, training, inference, RAG, agents, evaluation, and production architecture.", "topics": [], "category": "RAG Eval", "momentum": 42, "rank": 175 }, { "name": "ai-runtime-lab", "full_name": "Ali-hey-0/ai-runtime-lab", "slug": "ali-hey-0-ai-runtime-lab", "url": "https://github.com/Ali-hey-0/ai-runtime-lab", "owner": "Ali-hey-0", "owner_avatar": "https://avatars.githubusercontent.com/u/157505360?v=4", "stars": 183, "forks": 5, "open_issues": 8, "language": "Python", "license": "", "pushed_at": "2026-07-20T23:22:10Z", "created_at": "2026-07-20T22:58:03Z", "description": "Engineering deterministic, production-grade systems around non-deterministic LLMs — FSM, durable execution, retries, DAGs, agent runtimes, model routing, edge inference, RAG, memory, multi-agent orchestration, security, and observability. 14 runnable proof-of-concept phases.", "topics": [ "agent-architecture", "agent-memory", "agentic-ai", "ai-agents", "ai-security", "dag", "durable-execution", "finite-state-machine", "idempotency", "llm", "llm-evaluation", "model-routing", "multi-agent-systems", "observability", "production-ai", "prompt-injection", "rag", "retry-strategy", "vector-search", "workflow-engine" ], "category": "Observability", "momentum": 42, "rank": 176 }, { "name": "AgenticGoKit", "full_name": "AgenticGoKit/AgenticGoKit", "slug": "agenticgokit-agenticgokit", "url": "https://github.com/AgenticGoKit/AgenticGoKit", "owner": "AgenticGoKit", "owner_avatar": "https://avatars.githubusercontent.com/u/240036989?v=4", "stars": 182, "forks": 37, "open_issues": 33, "language": "Go", "license": "Apache-2.0", "pushed_at": "2026-08-01T03:56:57Z", "created_at": "2025-05-02T07:58:47Z", "description": "Open-source Agentic AI framework in Go for building, orchestrating, and deploying intelligent agents. LLM-agnostic, event-driven, with multi-agent workflows, MCP tool discovery, and production-grade observability.", "topics": [ "agentic-ai", "agentic-ai-development", "agentic-coding", "agentic-framework", "agentic-rag", "agentic-workflow", "agentic-workflows", "agents", "ai", "ai-agents", "ai-framework", "aiframework", "developer-tools", "framework", "golang", "mcp", "multi-agent", "multi-agent-systems" ], "category": "Observability", "momentum": 42, "rank": 177 }, { "name": "LLMZoo", "full_name": "FreedomIntelligence/LLMZoo", "slug": "freedomintelligence-llmzoo", "url": "https://github.com/FreedomIntelligence/LLMZoo", "owner": "FreedomIntelligence", "owner_avatar": "https://avatars.githubusercontent.com/u/127706844?v=4", "stars": 2934, "forks": 193, "open_issues": 26, "language": "Python", "license": "Apache-2.0", "pushed_at": "2023-11-26T17:48:30Z", "created_at": "2023-04-01T12:08:23Z", "description": "⚡LLM Zoo is a project that provides data, models, and evaluation benchmark for large language models.⚡", "topics": [], "category": "Benchmarks", "momentum": 41, "rank": 178 }, { "name": "MingLi-Bench", "full_name": "DestinyLinker/MingLi-Bench", "slug": "destinylinker-mingli-bench", "url": "https://github.com/DestinyLinker/MingLi-Bench", "owner": "DestinyLinker", "owner_avatar": "https://avatars.githubusercontent.com/u/227604965?v=4", "stars": 2544, "forks": 362, "open_issues": 3, "language": "Python", "license": "MIT", "pushed_at": "2026-05-09T17:49:34Z", "created_at": "2026-04-22T15:32:29Z", "description": "A benchmark for evaluating LLMs on Chinese traditional fortune telling — Bazi (八字) and Ziwei Doushu (紫微斗数).", "topics": [], "category": "Benchmarks", "momentum": 41, "rank": 179 }, { "name": "hub", "full_name": "traceloop/hub", "slug": "traceloop-hub", "url": "https://github.com/traceloop/hub", "owner": "traceloop", "owner_avatar": "https://avatars.githubusercontent.com/u/125419530?v=4", "stars": 233, "forks": 33, "open_issues": 12, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-07-16T09:13:24Z", "created_at": "2024-10-24T13:43:43Z", "description": "High-scale LLM gateway, written in Rust. OpenTelemetry-based observability included", "topics": [ "artificial-intelligence", "datascience", "generative-ai", "llm", "llmops", "ml", "model-monitoring", "observability", "open-source", "opentelemetry", "rust" ], "category": "Observability", "momentum": 41, "rank": 180 }, { "name": "oss-llmops-stack", "full_name": "langfuse/oss-llmops-stack", "slug": "langfuse-oss-llmops-stack", "url": "https://github.com/langfuse/oss-llmops-stack", "owner": "langfuse", "owner_avatar": "https://avatars.githubusercontent.com/u/134601687?v=4", "stars": 154, "forks": 8, "open_issues": 1, "language": "", "license": "MIT", "pushed_at": "2026-07-28T15:43:33Z", "created_at": "2025-02-08T16:59:06Z", "description": "Modular, open source LLMOps stack that separates concerns: LiteLLM unifies LLM APIs, manages routing and cost controls, and ensures high-availability, while Langfuse focuses on detailed observability, prompt versioning, and performance evaluations.", "topics": [ "ai-gateway", "gateway", "llm", "llm-evaluation", "llm-gateway", "llm-observability", "llmops", "monitoring", "open-source", "openai-proxy", "prompt-management", "self-hosted", "ycombinator" ], "category": "Observability", "momentum": 41, "rank": 181 }, { "name": "multivon-eval", "full_name": "multivon-ai/multivon-eval", "slug": "multivon-ai-multivon-eval", "url": "https://github.com/multivon-ai/multivon-eval", "owner": "multivon-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/279155390?v=4", "stars": 26, "forks": 0, "open_issues": 5, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-09-21T14:12:28Z", "created_at": "2026-04-24T08:45:51Z", "description": "Practical LLM evaluation for teams that ship to production. Deterministic + LLM-as-judge evaluators, dataset support, CI/CD integration.", "topics": [ "agent-evaluation", "ai-evaluation", "evals", "hallucination-detection", "llm-as-judge", "llm-eval", "llm-evaluation", "llmops", "mlops", "prompt-engineering", "python", "rag-evaluation" ], "category": "RAG Eval", "momentum": 41, "rank": 182 }, { "name": "uptrain", "full_name": "uptrain-ai/uptrain", "slug": "uptrain-ai-uptrain", "url": "https://github.com/uptrain-ai/uptrain", "owner": "uptrain-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/114582870?v=4", "stars": 2368, "forks": 206, "open_issues": 66, "language": "Python", "license": "Apache-2.0", "pushed_at": "2024-08-18T13:30:44Z", "created_at": "2022-11-07T07:29:06Z", "description": "UpTrain is an open-source unified platform to evaluate and improve Generative AI applications. We provide grades for 20+ preconfigured checks (covering language, code, embedding use-cases), perform root cause analysis on failure cases and give insights on how to resolve them.", "topics": [ "autoevaluation", "evaluation", "experimentation", "hallucination-detection", "jailbreak-detection", "llm-eval", "llm-prompting", "llm-test", "llmops", "machine-learning", "monitoring", "openai-evals", "prompt-engineering", "root-cause-analysis" ], "category": "Observability", "momentum": 40, "rank": 183 }, { "name": "graph-rag-agent", "full_name": "1517005260/graph-rag-agent", "slug": "1517005260-graph-rag-agent", "url": "https://github.com/1517005260/graph-rag-agent", "owner": "1517005260", "owner_avatar": "https://avatars.githubusercontent.com/u/113491113?v=4", "stars": 2356, "forks": 321, "open_issues": 38, "language": "Python", "license": "MIT", "pushed_at": "2025-11-05T08:15:51Z", "created_at": "2025-02-12T15:11:51Z", "description": "拼好RAG：手搓并融合了GraphRAG、LightRAG、Neo4j-llm-graph-builder进行知识图谱构建以及搜索；整合DeepSearch技术实现私域RAG的推理；自制针对GraphRAG的评估框架| Integrate GraphRAG, LightRAG, and Neo4j-llm-graph-builder for knowledge graph construction and search. Combine DeepSearch for private RAG reasoning. Create a custom evaluation framework for GraphRAG.", "topics": [ "agentic-rag", "chain-of-exploration", "deepresearch", "deepsearch", "evaluation", "graphrag", "graphsearch", "kg", "lightrag", "reasoning", "think-on-graph" ], "category": "RAG Eval", "momentum": 40, "rank": 184 }, { "name": "evaluation-guidebook", "full_name": "huggingface/evaluation-guidebook", "slug": "huggingface-evaluation-guidebook", "url": "https://github.com/huggingface/evaluation-guidebook", "owner": "huggingface", "owner_avatar": "https://avatars.githubusercontent.com/u/25720743?v=4", "stars": 2163, "forks": 124, "open_issues": 5, "language": "Jupyter Notebook", "license": "NOASSERTION", "pushed_at": "2025-12-03T14:45:05Z", "created_at": "2024-10-09T10:36:39Z", "description": "Sharing both practical insights and theoretical knowledge about LLM evaluation that we gathered while managing the Open LLM Leaderboard and designing lighteval!", "topics": [ "evaluation", "evaluation-metrics", "guidebook", "large-language-models", "llm", "machine-learning", "tutorial" ], "category": "Benchmarks", "momentum": 40, "rank": 185 }, { "name": "android-bench", "full_name": "android-bench/android-bench", "slug": "android-bench-android-bench", "url": "https://github.com/android-bench/android-bench", "owner": "android-bench", "owner_avatar": "https://avatars.githubusercontent.com/u/245893681?v=4", "stars": 301, "forks": 61, "open_issues": 14, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-07-08T15:30:44Z", "created_at": "2026-03-04T23:10:16Z", "description": "Android Bench is a framework for benchmarking Large Language Models (LLMs) on Android development tasks. It evaluates an AI model's ability to understand mobile codebases, generate accurate patches, and solve Android-specific engineering problems.", "topics": [], "category": "Benchmarks", "momentum": 40, "rank": 186 }, { "name": "Axon", "full_name": "langchain-tracer/Axon", "slug": "langchain-tracer-axon", "url": "https://github.com/langchain-tracer/Axon", "owner": "langchain-tracer", "owner_avatar": "https://avatars.githubusercontent.com/u/237815469?v=4", "stars": 93, "forks": 4, "open_issues": 0, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-08-06T17:30:37Z", "created_at": "2025-10-13T23:12:25Z", "description": "OpenTelemetry-native LLM observability CLI. Point any OTEL exporter at it and watch your LLM/agent traces in real time.", "topics": [ "agents", "ai-agents", "langchain", "langraph", "llm", "llm-ops", "observability", "opentelemetry", "otel", "tracing", "typescript" ], "category": "Observability", "momentum": 40, "rank": 187 }, { "name": "GPU-Benchmarks-on-LLM-Inference", "full_name": "XiongjieDai/GPU-Benchmarks-on-LLM-Inference", "slug": "xiongjiedai-gpu-benchmarks-on-llm-inference", "url": "https://github.com/XiongjieDai/GPU-Benchmarks-on-LLM-Inference", "owner": "XiongjieDai", "owner_avatar": "https://avatars.githubusercontent.com/u/53258114?v=4", "stars": 1941, "forks": 76, "open_issues": 15, "language": "Jupyter Notebook", "license": "", "pushed_at": "2024-05-13T00:29:05Z", "created_at": "2023-07-19T01:53:11Z", "description": "Multiple NVIDIA GPUs or Apple Silicon for Large Language Model Inference?", "topics": [], "category": "Benchmarks", "momentum": 39, "rank": 188 }, { "name": "OpenRedTeaming", "full_name": "Libr-AI/OpenRedTeaming", "slug": "libr-ai-openredteaming", "url": "https://github.com/Libr-AI/OpenRedTeaming", "owner": "Libr-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/133515165?v=4", "stars": 182, "forks": 17, "open_issues": 3, "language": "", "license": "MIT", "pushed_at": "2026-07-14T15:40:55Z", "created_at": "2024-02-05T07:16:00Z", "description": "Papers about red teaming LLMs and Multimodal models.", "topics": [ "awesome-list", "language-model", "llm-safety", "papers", "redteaming", "safety" ], "category": "Red Teaming & Safety", "momentum": 39, "rank": 189 }, { "name": "trench", "full_name": "FrigadeHQ/trench", "slug": "frigadehq-trench", "url": "https://github.com/FrigadeHQ/trench", "owner": "FrigadeHQ", "owner_avatar": "https://avatars.githubusercontent.com/u/117952255?v=4", "stars": 1666, "forks": 66, "open_issues": 3, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-04-06T23:11:33Z", "created_at": "2024-09-30T19:19:01Z", "description": "Trench — Open-Source Analytics Infrastructure. A single production-ready Docker image built on ClickHouse, Kafka, and Node.js for tracking events. Easily build product analytics dashboards, LLM RAGs, observability platforms, or any other analytics product.", "topics": [ "analytics", "clickhouse", "clickhouse-database", "clickhouse-server", "dashboard", "dashboards", "event-replay", "kafka", "kafka-engine", "llm", "matomo", "matomo-analytics", "nodejs", "open-source", "plausible-analytics", "posthog", "product-analytics", "rag", "tracking", "tracking-events" ], "category": "Observability", "momentum": 38, "rank": 190 }, { "name": "FuzzyAI", "full_name": "cyberark/FuzzyAI", "slug": "cyberark-fuzzyai", "url": "https://github.com/cyberark/FuzzyAI", "owner": "cyberark", "owner_avatar": "https://avatars.githubusercontent.com/u/30869256?v=4", "stars": 1599, "forks": 219, "open_issues": 6, "language": "Jupyter Notebook", "license": "Apache-2.0", "pushed_at": "2026-02-06T21:59:21Z", "created_at": "2024-12-03T11:27:53Z", "description": "A powerful tool for automated LLM fuzzing. It is designed to help developers and security researchers identify and mitigate potential jailbreaks in their LLM APIs.", "topics": [ "ai", "ai-red-team", "fuzzing", "jailbreak", "jailbreaking", "llm", "llm-evaluation", "llm-security", "llms", "security" ], "category": "Red Teaming & Safety", "momentum": 38, "rank": 191 }, { "name": "llm-colosseum", "full_name": "OpenGenerativeAI/llm-colosseum", "slug": "opengenerativeai-llm-colosseum", "url": "https://github.com/OpenGenerativeAI/llm-colosseum", "owner": "OpenGenerativeAI", "owner_avatar": "https://avatars.githubusercontent.com/u/138796527?v=4", "stars": 1485, "forks": 180, "open_issues": 16, "language": "Jupyter Notebook", "license": "MIT", "pushed_at": "2025-03-21T10:35:26Z", "created_at": "2024-03-23T05:52:20Z", "description": "Benchmark LLMs by fighting in Street Fighter 3! The new way to evaluate the quality of an LLM", "topics": [ "benchmark", "genai", "llm", "streetfighterai" ], "category": "Benchmarks", "momentum": 38, "rank": 192 }, { "name": "GraphRAG-Benchmark", "full_name": "GraphRAG-Bench/GraphRAG-Benchmark", "slug": "graphrag-bench-graphrag-benchmark", "url": "https://github.com/GraphRAG-Bench/GraphRAG-Benchmark", "owner": "GraphRAG-Bench", "owner_avatar": "https://avatars.githubusercontent.com/u/211924730?v=4", "stars": 518, "forks": 59, "open_issues": 11, "language": "Python", "license": "MIT", "pushed_at": "2026-06-07T02:17:59Z", "created_at": "2025-06-09T03:34:06Z", "description": "The official repo of GraphRAG-Bench for evaluating GraphRAG models. \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation\". (ICLR'26)", "topics": [ "analysis", "benchmark", "dataset", "graphrag", "graphrag-bench", "graphrag-benchmark", "graphragdataset" ], "category": "RAG Eval", "momentum": 38, "rank": 193 }, { "name": "openllmetry-js", "full_name": "traceloop/openllmetry-js", "slug": "traceloop-openllmetry-js", "url": "https://github.com/traceloop/openllmetry-js", "owner": "traceloop", "owner_avatar": "https://avatars.githubusercontent.com/u/125419530?v=4", "stars": 411, "forks": 77, "open_issues": 121, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2026-06-18T14:49:18Z", "created_at": "2023-09-27T22:38:28Z", "description": "Sister project to OpenLLMetry, but in Typescript. Open-source observability for your LLM application, based on OpenTelemetry", "topics": [ "datascience", "generative-ai", "javascript", "llmops", "metrics", "ml", "model-monitoring", "monitoring", "nextjs", "observability", "open-source", "opentelemetry", "opentelemetry-javascript", "typescript" ], "category": "Observability", "momentum": 38, "rank": 194 }, { "name": "SWELancer-Benchmark", "full_name": "openai/SWELancer-Benchmark", "slug": "openai-swelancer-benchmark", "url": "https://github.com/openai/SWELancer-Benchmark", "owner": "openai", "owner_avatar": "https://avatars.githubusercontent.com/u/14957082?v=4", "stars": 1430, "forks": 135, "open_issues": 0, "language": "", "license": "", "pushed_at": "2025-07-18T02:02:20Z", "created_at": "2025-02-18T17:23:15Z", "description": "This repo contains the dataset and code for the paper \"SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?\"", "topics": [], "category": "Coding Eval", "momentum": 37, "rank": 195 }, { "name": "text-to-lora", "full_name": "SakanaAI/text-to-lora", "slug": "sakanaai-text-to-lora", "url": "https://github.com/SakanaAI/text-to-lora", "owner": "SakanaAI", "owner_avatar": "https://avatars.githubusercontent.com/u/140988036?v=4", "stars": 1328, "forks": 93, "open_issues": 3, "language": "Python", "license": "Apache-2.0", "pushed_at": "2025-06-08T14:42:10Z", "created_at": "2025-05-05T09:11:35Z", "description": "Hypernetworks that adapt LLMs for specific benchmark tasks using only textual task description as the input", "topics": [ "fine-tuning", "hypernetworks", "llm", "lora", "machine-learning" ], "category": "Benchmarks", "momentum": 37, "rank": 196 }, { "name": "KernelBench", "full_name": "ScalingIntelligence/KernelBench", "slug": "scalingintelligence-kernelbench", "url": "https://github.com/ScalingIntelligence/KernelBench", "owner": "ScalingIntelligence", "owner_avatar": "https://avatars.githubusercontent.com/u/171742855?v=4", "stars": 1314, "forks": 196, "open_issues": 58, "language": "Jupyter Notebook", "license": "NOASSERTION", "pushed_at": "2026-03-24T04:29:13Z", "created_at": "2024-10-25T00:08:19Z", "description": "KernelBench: Can LLMs Write GPU Kernels? - Benchmark + Toolkit with Torch -> CUDA (+ more DSLs)", "topics": [ "benchmark", "codegen", "evaluation", "gpu", "rl-environment", "tooling" ], "category": "Benchmarks", "momentum": 37, "rank": 197 }, { "name": "langtrace", "full_name": "Scale3-Labs/langtrace", "slug": "scale3-labs-langtrace", "url": "https://github.com/Scale3-Labs/langtrace", "owner": "Scale3-Labs", "owner_avatar": "https://avatars.githubusercontent.com/u/110545750?v=4", "stars": 1233, "forks": 128, "open_issues": 2, "language": "TypeScript", "license": "AGPL-3.0", "pushed_at": "2025-11-17T15:08:48Z", "created_at": "2024-03-30T23:49:41Z", "description": "Langtrace 🔍 is an open-source, Open Telemetry based end-to-end observability tool for LLM applications, providing real-time tracing, evaluations and metrics for popular LLMs, LLM frameworks, vectorDBs and more.. Integrate using Typescript, Python. 🚀💻📊", "topics": [ "ai", "datasets", "evaluations", "gpt", "langchain", "llm", "llm-framework", "llmops", "observability", "open-source", "open-telemetry", "openai", "prompt-engineering", "tracing" ], "category": "Observability", "momentum": 37, "rank": 198 }, { "name": "promptly", "full_name": "equinor/promptly", "slug": "equinor-promptly", "url": "https://github.com/equinor/promptly", "owner": "equinor", "owner_avatar": "https://avatars.githubusercontent.com/u/525862?v=4", "stars": 30, "forks": 3, "open_issues": 0, "language": "Jupyter Notebook", "license": "CC-BY-4.0", "pushed_at": "2026-08-24T17:56:31Z", "created_at": "2024-02-12T08:08:15Z", "description": "A prompt collection for testing and evaluation of LLMs.", "topics": [ "chatgpt", "llm-evaluation", "prompt-engineering", "prompts" ], "category": "Eval Frameworks", "momentum": 37, "rank": 199 }, { "name": "llmperf", "full_name": "ray-project/llmperf", "slug": "ray-project-llmperf", "url": "https://github.com/ray-project/llmperf", "owner": "ray-project", "owner_avatar": "https://avatars.githubusercontent.com/u/22125274?v=4", "stars": 1127, "forks": 202, "open_issues": 55, "language": "Python", "license": "Apache-2.0", "pushed_at": "2024-12-09T01:52:28Z", "created_at": "2023-10-10T16:49:58Z", "description": "LLMPerf is a library for validating and benchmarking LLMs", "topics": [], "category": "Benchmarks", "momentum": 36, "rank": 200 }, { "name": "prometheus-eval", "full_name": "prometheus-eval/prometheus-eval", "slug": "prometheus-eval-prometheus-eval", "url": "https://github.com/prometheus-eval/prometheus-eval", "owner": "prometheus-eval", "owner_avatar": "https://avatars.githubusercontent.com/u/167460660?v=4", "stars": 1122, "forks": 68, "open_issues": 13, "language": "Python", "license": "Apache-2.0", "pushed_at": "2025-04-25T03:58:37Z", "created_at": "2024-04-18T17:19:09Z", "description": "Evaluate your LLM's response with Prometheus and GPT4 💯", "topics": [ "evaluation", "gpt4", "litellm", "llm", "llm-as-a-judge", "llm-as-evaluator", "llmops", "python", "vllm" ], "category": "Eval Frameworks", "momentum": 36, "rank": 201 }, { "name": "arena-hard-auto", "full_name": "lmarena/arena-hard-auto", "slug": "lmarena-arena-hard-auto", "url": "https://github.com/lmarena/arena-hard-auto", "owner": "lmarena", "owner_avatar": "https://avatars.githubusercontent.com/u/177094203?v=4", "stars": 1087, "forks": 163, "open_issues": 21, "language": "Python", "license": "Apache-2.0", "pushed_at": "2025-06-21T04:44:12Z", "created_at": "2023-11-04T03:54:08Z", "description": "Arena-Hard-Auto: An automatic LLM benchmark.", "topics": [], "category": "Benchmarks", "momentum": 36, "rank": 202 }, { "name": "yet-another-applied-llm-benchmark", "full_name": "carlini/yet-another-applied-llm-benchmark", "slug": "carlini-yet-another-applied-llm-benchmark", "url": "https://github.com/carlini/yet-another-applied-llm-benchmark", "owner": "carlini", "owner_avatar": "https://avatars.githubusercontent.com/u/1269300?v=4", "stars": 1070, "forks": 78, "open_issues": 11, "language": "Python", "license": "GPL-3.0", "pushed_at": "2025-04-27T06:25:09Z", "created_at": "2023-12-18T02:06:35Z", "description": "A benchmark to evaluate language models on questions I've previously asked them to solve.", "topics": [], "category": "Benchmarks", "momentum": 36, "rank": 203 }, { "name": "Mind2Web", "full_name": "OSU-NLP-Group/Mind2Web", "slug": "osu-nlp-group-mind2web", "url": "https://github.com/OSU-NLP-Group/Mind2Web", "owner": "OSU-NLP-Group", "owner_avatar": "https://avatars.githubusercontent.com/u/92067480?v=4", "stars": 1036, "forks": 123, "open_issues": 12, "language": "Jupyter Notebook", "license": "MIT", "pushed_at": "2025-11-05T00:38:41Z", "created_at": "2023-04-21T17:46:15Z", "description": "[NeurIPS'23 Spotlight] \"Mind2Web: Towards a Generalist Agent for the Web\" -- the first LLM-based web agent and benchmark for generalist web agents", "topics": [], "category": "Agent Eval", "momentum": 36, "rank": 204 }, { "name": "langkit", "full_name": "whylabs/langkit", "slug": "whylabs-langkit", "url": "https://github.com/whylabs/langkit", "owner": "whylabs", "owner_avatar": "https://avatars.githubusercontent.com/u/56651354?v=4", "stars": 1005, "forks": 77, "open_issues": 39, "language": "Jupyter Notebook", "license": "Apache-2.0", "pushed_at": "2024-11-22T20:02:14Z", "created_at": "2023-04-26T21:46:58Z", "description": "🔍 LangKit: An open-source toolkit for monitoring Large Language Models (LLMs). 📚 Extracts signals from prompts & responses, ensuring safety & security. 🛡️ Features include text quality, relevance metrics, & sentiment analysis. 📊 A comprehensive tool for LLM observability. 👀", "topics": [ "large-language-models", "machine-learning", "nlg", "nlp", "observability", "prompt-engineering", "prompt-injection" ], "category": "Observability", "momentum": 36, "rank": 205 }, { "name": "ATLAS", "full_name": "VILA-Lab/ATLAS", "slug": "vila-lab-atlas", "url": "https://github.com/VILA-Lab/ATLAS", "owner": "VILA-Lab", "owner_avatar": "https://avatars.githubusercontent.com/u/101944607?v=4", "stars": 991, "forks": 106, "open_issues": 7, "language": "Python", "license": "Apache-2.0", "pushed_at": "2024-05-28T05:52:07Z", "created_at": "2023-12-08T12:46:53Z", "description": "A principled instruction benchmark on formulating effective queries and prompts for large language models (LLMs). Our paper: https://arxiv.org/abs/2312.16171", "topics": [], "category": "Benchmarks", "momentum": 36, "rank": 206 }, { "name": "moonshot", "full_name": "aiverify-foundation/moonshot", "slug": "aiverify-foundation-moonshot", "url": "https://github.com/aiverify-foundation/moonshot", "owner": "aiverify-foundation", "owner_avatar": "https://avatars.githubusercontent.com/u/118954826?v=4", "stars": 360, "forks": 73, "open_issues": 4, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-06-10T03:26:13Z", "created_at": "2023-12-14T07:43:03Z", "description": "Moonshot - A simple and modular tool to evaluate and red-team any LLM application.", "topics": [ "benchmarking", "evaluation-framework", "llm", "red-teaming", "trustworthy-ai" ], "category": "Red Teaming & Safety", "momentum": 36, "rank": 207 }, { "name": "cryptex-oss", "full_name": "m4xx101/cryptex-oss", "slug": "m4xx101-cryptex-oss", "url": "https://github.com/m4xx101/cryptex-oss", "owner": "m4xx101", "owner_avatar": "https://avatars.githubusercontent.com/u/46222738?v=4", "stars": 338, "forks": 63, "open_issues": 0, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-06-09T19:49:30Z", "created_at": "2026-05-08T23:58:48Z", "description": "Open-source LLM red-teaming technique toolkit (162 transforms, 36 mutators, 25 tool surfaces). MIT.", "topics": [], "category": "Red Teaming & Safety", "momentum": 36, "rank": 208 }, { "name": "awesome-red-teaming-llms", "full_name": "dapurv5/awesome-red-teaming-llms", "slug": "dapurv5-awesome-red-teaming-llms", "url": "https://github.com/dapurv5/awesome-red-teaming-llms", "owner": "dapurv5", "owner_avatar": "https://avatars.githubusercontent.com/u/654346?v=4", "stars": 52, "forks": 14, "open_issues": 2, "language": "", "license": "CC-BY-4.0", "pushed_at": "2026-08-05T04:07:28Z", "created_at": "2024-04-27T01:29:47Z", "description": "Papers from our SoK on Red-Teaming (Accepted at TMLR)", "topics": [ "adversarial-attacks", "ai-safety", "ai-security", "awesome", "awesome-list", "llm-safety", "llm-security", "red-teaming" ], "category": "Red Teaming & Safety", "momentum": 36, "rank": 209 }, { "name": "nexa-gauge", "full_name": "harnexa/nexa-gauge", "slug": "harnexa-nexa-gauge", "url": "https://github.com/harnexa/nexa-gauge", "owner": "harnexa", "owner_avatar": "https://avatars.githubusercontent.com/u/277560842?v=4", "stars": 40, "forks": 14, "open_issues": 3, "language": "Python", "license": "MIT", "pushed_at": "2026-08-03T09:25:37Z", "created_at": "2026-03-12T06:26:36Z", "description": "An graph-eval framework for LLM's", "topics": [ "blue-score", "geval", "grounding", "llm", "llm-eval", "llm-evaluation-framework", "llm-judge", "ranking-algorithm", "redteam", "relevance-scoring", "rouge-metric" ], "category": "Red Teaming & Safety", "momentum": 35, "rank": 210 }, { "name": "web-codegen-scorer", "full_name": "angular/web-codegen-scorer", "slug": "angular-web-codegen-scorer", "url": "https://github.com/angular/web-codegen-scorer", "owner": "angular", "owner_avatar": "https://avatars.githubusercontent.com/u/139426?v=4", "stars": 771, "forks": 66, "open_issues": 20, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-05-05T19:03:44Z", "created_at": "2025-09-04T18:57:45Z", "description": "Web Codegen Scorer is a tool for evaluating the quality of web code generated by LLMs.", "topics": [ "benchmarking", "codegen", "evaluation", "llm-coding" ], "category": "Coding Eval", "momentum": 34, "rank": 211 }, { "name": "contoso-chat", "full_name": "Azure-Samples/contoso-chat", "slug": "azure-samples-contoso-chat", "url": "https://github.com/Azure-Samples/contoso-chat", "owner": "Azure-Samples", "owner_avatar": "https://avatars.githubusercontent.com/u/1844662?v=4", "stars": 766, "forks": 3931, "open_issues": 25, "language": "Bicep", "license": "MIT", "pushed_at": "2025-10-03T13:51:14Z", "created_at": "2023-11-29T19:10:30Z", "description": "This sample has the full End2End process of creating RAG application with Prompty and Azure AI Foundry. It includes GPT-4 LLM application code, evaluations, deployment automation with AZD CLI, GitHub actions for evaluation and deployment and intent mapping for multiple LLM task mapping.", "topics": [ "ai-azd-templates", "azd-templates", "azure-ai-foundry", "azure-ai-samples", "azure-ai-search", "azure-ai-services", "azure-cosmosdb", "azure-openai-service", "contoso-chat", "copilot-tutorial", "llmops", "rag-copilot", "retail-sample" ], "category": "RAG Eval", "momentum": 34, "rank": 212 }, { "name": "ARES", "full_name": "stanford-futuredata/ARES", "slug": "stanford-futuredata-ares", "url": "https://github.com/stanford-futuredata/ARES", "owner": "stanford-futuredata", "owner_avatar": "https://avatars.githubusercontent.com/u/16332752?v=4", "stars": 741, "forks": 67, "open_issues": 23, "language": "Python", "license": "Apache-2.0", "pushed_at": "2025-03-28T11:25:50Z", "created_at": "2023-09-27T03:56:19Z", "description": "Automated Evaluation of RAG Systems", "topics": [], "category": "RAG Eval", "momentum": 34, "rank": 213 }, { "name": "Awesome-LLM-Eval", "full_name": "onejune2018/Awesome-LLM-Eval", "slug": "onejune2018-awesome-llm-eval", "url": "https://github.com/onejune2018/Awesome-LLM-Eval", "owner": "onejune2018", "owner_avatar": "https://avatars.githubusercontent.com/u/35061855?v=4", "stars": 659, "forks": 111, "open_issues": 71, "language": "", "license": "MIT", "pushed_at": "2025-11-24T01:59:12Z", "created_at": "2023-04-26T10:10:06Z", "description": "Awesome-LLM-Eval: a curated list of tools, datasets/benchmark, demos, leaderboard, papers, docs and models, mainly for Evaluation on LLMs. 一个由工具、基准/数据、演示、排行榜和大模型等组成的精选列表，主要面向基础大模型评测，旨在探求生成式AI的技术边界.", "topics": [ "awsome-list", "awsome-lists", "benchmark", "bert", "chatglm", "chatgpt", "dataset", "evaluation", "gpt3", "large-language-model", "leaderboard", "llama", "llm", "llm-evaluation", "machine-learning", "nlp", "openai", "qwen", "rag" ], "category": "RAG Eval", "momentum": 33, "rank": 214 }, { "name": "RAG-Driven-Generative-AI", "full_name": "Denis2054/RAG-Driven-Generative-AI", "slug": "denis2054-rag-driven-generative-ai", "url": "https://github.com/Denis2054/RAG-Driven-Generative-AI", "owner": "Denis2054", "owner_avatar": "https://avatars.githubusercontent.com/u/30811222?v=4", "stars": 631, "forks": 224, "open_issues": 0, "language": "Jupyter Notebook", "license": "MIT", "pushed_at": "2025-09-23T15:31:25Z", "created_at": "2024-04-19T17:40:16Z", "description": "This repository provides programs to build Retrieval Augmented Generation (RAG) code for Generative AI with LlamaIndex, Deep Lake, and Pinecone leveraging the power of OpenAI and Hugging Face models for generation and evaluation.", "topics": [ "advanced-rag", "chroma", "chromadb", "embedding-models", "fine-tuning", "gpt-4o-mini", "gpt4-omni", "grok", "huggingface", "indexing-querying", "llama", "llama-index", "multimodal", "openai-api", "pinecone", "rag", "scaling", "vision-transformer", "xai-grok" ], "category": "RAG Eval", "momentum": 33, "rank": 215 }, { "name": "DVAIA-Damn-Vulnerable-AI-Application", "full_name": "airtasystems/DVAIA-Damn-Vulnerable-AI-Application", "slug": "airtasystems-dvaia-damn-vulnerable-ai-application", "url": "https://github.com/airtasystems/DVAIA-Damn-Vulnerable-AI-Application", "owner": "airtasystems", "owner_avatar": "https://avatars.githubusercontent.com/u/158203648?v=4", "stars": 46, "forks": 21, "open_issues": 1, "language": "Python", "license": "", "pushed_at": "2026-07-21T23:18:24Z", "created_at": "2026-02-12T09:19:35Z", "description": "Damn Vulnerable AI Application - For LLM Red Team Training. LLM testing, RAG testing, Multimodal testing, Agent testing, LLM paload generation", "topics": [ "ai-security", "ai-testing", "ai-testing-tool", "dvwa", "llm-red-teaming", "mlops" ], "category": "Red Teaming & Safety", "momentum": 33, "rank": 216 }, { "name": "sample-getting-started-with-strands-agents-course", "full_name": "aws-samples/sample-getting-started-with-strands-agents-course", "slug": "aws-samples-sample-getting-started-with-strands-agents-course", "url": "https://github.com/aws-samples/sample-getting-started-with-strands-agents-course", "owner": "aws-samples", "owner_avatar": "https://avatars.githubusercontent.com/u/8931462?v=4", "stars": 196, "forks": 80, "open_issues": 4, "language": "Jupyter Notebook", "license": "MIT-0", "pushed_at": "2026-06-05T21:55:57Z", "created_at": "2025-08-05T19:18:08Z", "description": "Learn to build AI agents with Strands framework. Covers LLM integration via Amazon Bedrock/Anthropic, AWS service connections, tool implementation with MCP/A2A protocols, and agent evaluation using LangFuse/RAGAS.", "topics": [ "a2a", "anthropic", "aws", "bedrock", "litellm", "mcp", "python", "strands-agents" ], "category": "Observability", "momentum": 32, "rank": 217 }, { "name": "ai-llm-red-team-handbook", "full_name": "Shiva108/ai-llm-red-team-handbook", "slug": "shiva108-ai-llm-red-team-handbook", "url": "https://github.com/Shiva108/ai-llm-red-team-handbook", "owner": "Shiva108", "owner_avatar": "https://avatars.githubusercontent.com/u/13223532?v=4", "stars": 343, "forks": 65, "open_issues": 0, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-05-08T10:22:40Z", "created_at": "2025-11-27T08:36:30Z", "description": "AI / LLM Red Team Field Manual & Consultant’s Handbook", "topics": [], "category": "Red Teaming & Safety", "momentum": 30, "rank": 218 }, { "name": "HackSynth", "full_name": "aielte-research/HackSynth", "slug": "aielte-research-hacksynth", "url": "https://github.com/aielte-research/HackSynth", "owner": "aielte-research", "owner_avatar": "https://avatars.githubusercontent.com/u/61852660?v=4", "stars": 321, "forks": 50, "open_issues": 0, "language": "Python", "license": "AGPL-3.0", "pushed_at": "2025-06-24T10:20:29Z", "created_at": "2024-11-27T16:02:49Z", "description": "LLM Agent and Evaluation Framework for Autonomous Penetration Testing", "topics": [ "ai", "autonomous-pentesting", "ctf", "ctf-tools", "cybersecurity", "llms", "penetration-testing" ], "category": "Agent Eval", "momentum": 30, "rank": 219 }, { "name": "AI-penetration-testing", "full_name": "Mr-Infect/AI-penetration-testing", "slug": "mr-infect-ai-penetration-testing", "url": "https://github.com/Mr-Infect/AI-penetration-testing", "owner": "Mr-Infect", "owner_avatar": "https://avatars.githubusercontent.com/u/177466722?v=4", "stars": 311, "forks": 60, "open_issues": 0, "language": "", "license": "", "pushed_at": "2026-02-19T14:21:05Z", "created_at": "2025-06-12T14:43:48Z", "description": "AI/ML/LLM Penetration Testing Toolkit by Mr-Infect — the #1 GitHub resource for AI security, red teaming, and adversarial ML techniques. This repository is dedicated to offensive and defensive security for cutting-edge AI, Machine Learning (ML), and Large Language Models (LLMs) like ChatGPT, Claude, and LLaMA.", "topics": [ "ai", "aipentest", "automation", "cybersecurity", "ethical-hacking", "penetration-testing", "security-tools", "vulnerability-assessment" ], "category": "Red Teaming & Safety", "momentum": 30, "rank": 220 }, { "name": "athina-evals", "full_name": "athina-ai/athina-evals", "slug": "athina-ai-athina-evals", "url": "https://github.com/athina-ai/athina-evals", "owner": "athina-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/139258696?v=4", "stars": 302, "forks": 23, "open_issues": 6, "language": "Python", "license": "", "pushed_at": "2025-06-06T15:54:38Z", "created_at": "2023-11-22T10:46:44Z", "description": "Python SDK for running evaluations on LLM generated responses", "topics": [ "evaluation", "evaluation-framework", "evaluation-metrics", "llm-eval", "llm-evaluation", "llm-evaluation-toolkit", "llm-ops", "llmops" ], "category": "Eval Frameworks", "momentum": 29, "rank": 221 }, { "name": "RAGTune", "full_name": "misbahsy/RAGTune", "slug": "misbahsy-ragtune", "url": "https://github.com/misbahsy/RAGTune", "owner": "misbahsy", "owner_avatar": "https://avatars.githubusercontent.com/u/35427888?v=4", "stars": 261, "forks": 22, "open_issues": 4, "language": "Python", "license": "MIT", "pushed_at": "2024-03-19T18:37:42Z", "created_at": "2024-03-04T21:14:26Z", "description": "Tuning and Evaluation of RAG pipeline. (Automated optimization to be added soon)", "topics": [], "category": "RAG Eval", "momentum": 29, "rank": 222 }, { "name": "chatgpt-web-sea", "full_name": "SuYxh/chatgpt-web-sea", "slug": "suyxh-chatgpt-web-sea", "url": "https://github.com/SuYxh/chatgpt-web-sea", "owner": "SuYxh", "owner_avatar": "https://avatars.githubusercontent.com/u/46957352?v=4", "stars": 256, "forks": 52, "open_issues": 1, "language": "Vue", "license": "MIT", "pushed_at": "2024-05-14T01:39:53Z", "created_at": "2024-04-16T01:27:16Z", "description": "基于 chatgpt-web 兼容 LLM Red Team 项目下所有的 free-api 以及 one-api 或 new-api提供的接口和 key", "topics": [], "category": "Red Teaming & Safety", "momentum": 29, "rank": 223 }, { "name": "redteam-ai-benchmark", "full_name": "toxy4ny/redteam-ai-benchmark", "slug": "toxy4ny-redteam-ai-benchmark", "url": "https://github.com/toxy4ny/redteam-ai-benchmark", "owner": "toxy4ny", "owner_avatar": "https://avatars.githubusercontent.com/u/199085504?v=4", "stars": 61, "forks": 5, "open_issues": 2, "language": "Python", "license": "MIT", "pushed_at": "2026-06-22T12:06:38Z", "created_at": "2025-11-17T10:29:25Z", "description": "Red Team AI Benchmark: Evaluating Uncensored LLMs for Offensive Security", "topics": [ "ai", "ai-agents", "cybersecurity", "cybersecurity-tools", "hacking", "hacktools", "mlsecops", "prompt-engineering", "prompt-tuning", "rag", "rag-chatbot", "redteam", "redteaming" ], "category": "Red Teaming & Safety", "momentum": 29, "rank": 224 }, { "name": "evalcore", "full_name": "Eval-core/evalcore", "slug": "eval-core-evalcore", "url": "https://github.com/Eval-core/evalcore", "owner": "Eval-core", "owner_avatar": "https://avatars.githubusercontent.com/u/306051663?v=4", "stars": 17, "forks": 3, "open_issues": 11, "language": "Rust", "license": "Apache-2.0", "pushed_at": "2026-07-26T05:18:25Z", "created_at": "2026-07-17T03:26:45Z", "description": "Snapshot testing for LLM apps and agents, built to run locally and block regressions in CI.", "topics": [ "ci", "ci-cd", "cicd", "evaluation", "evaluation-framework", "llm", "llm-eval", "llm-evaluation", "llm-evaluation-framework", "llmops", "prompt-engineering", "prompt-testing", "prompts", "rag", "testing" ], "category": "RAG Eval", "momentum": 29, "rank": 225 }, { "name": "llm-bulls-and-cows-benchmark", "full_name": "stalkermustang/llm-bulls-and-cows-benchmark", "slug": "stalkermustang-llm-bulls-and-cows-benchmark", "url": "https://github.com/stalkermustang/llm-bulls-and-cows-benchmark", "owner": "stalkermustang", "owner_avatar": "https://avatars.githubusercontent.com/u/26546178?v=4", "stars": 235, "forks": 2, "open_issues": 0, "language": "HTML", "license": "MIT", "pushed_at": "2025-01-31T20:19:23Z", "created_at": "2024-11-26T07:58:12Z", "description": "A mini-framework for evaluating LLM performance on the Bulls and Cows number guessing game, supporting multiple LLM providers.", "topics": [ "benchmark", "benchmarking", "chatgpt", "games", "llm", "openai", "python", "reasoning" ], "category": "Reasoning", "momentum": 28, "rank": 226 }, { "name": "rag-arena", "full_name": "firecrawl/rag-arena", "slug": "firecrawl-rag-arena", "url": "https://github.com/firecrawl/rag-arena", "owner": "firecrawl", "owner_avatar": "https://avatars.githubusercontent.com/u/135057108?v=4", "stars": 222, "forks": 36, "open_issues": 3, "language": "TypeScript", "license": "MIT", "pushed_at": "2024-04-14T16:53:41Z", "created_at": "2024-02-17T23:48:49Z", "description": "Open-source RAG evaluation through users' feedback", "topics": [ "ai", "chatbot", "langchain", "langchain-typescript", "mendable", "nextjs", "rag", "retrieval" ], "category": "RAG Eval", "momentum": 28, "rank": 227 }, { "name": "eval-dev-quality", "full_name": "symflower/eval-dev-quality", "slug": "symflower-eval-dev-quality", "url": "https://github.com/symflower/eval-dev-quality", "owner": "symflower", "owner_avatar": "https://avatars.githubusercontent.com/u/20767533?v=4", "stars": 186, "forks": 10, "open_issues": 25, "language": "Go", "license": "MIT", "pushed_at": "2025-05-15T19:26:46Z", "created_at": "2024-03-28T17:09:34Z", "description": "DevQualityEval: An evaluation benchmark 📈 and framework to compare and evolve the quality of code generation of LLMs.", "topics": [ "evaluation", "evaluation-framework", "llms", "software-development", "software-quality" ], "category": "Coding Eval", "momentum": 27, "rank": 228 }, { "name": "ParroT", "full_name": "wxjiao/ParroT", "slug": "wxjiao-parrot", "url": "https://github.com/wxjiao/ParroT", "owner": "wxjiao", "owner_avatar": "https://avatars.githubusercontent.com/u/31032829?v=4", "stars": 177, "forks": 23, "open_issues": 4, "language": "Python", "license": "", "pushed_at": "2024-12-31T03:51:23Z", "created_at": "2023-03-22T03:19:42Z", "description": "The ParroT framework to enhance and regulate the Translation Abilities during Chat based on open-sourced LLMs (e.g., LLaMA-7b, Bloomz-7b1-mt) and human written translation and evaluation data.", "topics": [ "bloomz", "chatgpt", "contrastive", "error-guided", "gpt-4", "human-feedback", "instruction-tuning", "llama", "lora", "machine-translation" ], "category": "Eval Frameworks", "momentum": 27, "rank": 229 }, { "name": "artkit", "full_name": "BCG-X-Official/artkit", "slug": "bcg-x-official-artkit", "url": "https://github.com/BCG-X-Official/artkit", "owner": "BCG-X-Official", "owner_avatar": "https://avatars.githubusercontent.com/u/70293226?v=4", "stars": 174, "forks": 38, "open_issues": 10, "language": "Jupyter Notebook", "license": "Apache-2.0", "pushed_at": "2025-03-06T18:33:04Z", "created_at": "2024-06-09T15:15:39Z", "description": "Automated prompt-based testing and evaluation of Gen AI applications", "topics": [ "asyncio", "data-science", "gen-ai", "genai", "python", "red-teaming", "test-automation" ], "category": "Red Teaming & Safety", "momentum": 27, "rank": 230 }, { "name": "rageval", "full_name": "gomate-community/rageval", "slug": "gomate-community-rageval", "url": "https://github.com/gomate-community/rageval", "owner": "gomate-community", "owner_avatar": "https://avatars.githubusercontent.com/u/158247319?v=4", "stars": 172, "forks": 10, "open_issues": 22, "language": "Python", "license": "Apache-2.0", "pushed_at": "2024-11-18T09:16:08Z", "created_at": "2024-01-30T07:27:32Z", "description": "Evaluation tools for Retrieval-augmented Generation (RAG) methods.", "topics": [ "evalution", "llm", "rag" ], "category": "RAG Eval", "momentum": 27, "rank": 231 }, { "name": "skill-optimizer", "full_name": "fastxyz/skill-optimizer", "slug": "fastxyz-skill-optimizer", "url": "https://github.com/fastxyz/skill-optimizer", "owner": "fastxyz", "owner_avatar": "https://avatars.githubusercontent.com/u/153221483?v=4", "stars": 83, "forks": 14, "open_issues": 15, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-05-28T17:39:51Z", "created_at": "2026-03-06T10:34:42Z", "description": "Benchmark, evaluate, and optimize skills to ensure reliable performance across all LLMs", "topics": [ "ai", "ai-agent", "ai-skill", "benchmark", "cli", "eval", "evals", "evaluation", "evaluation-framework", "llm", "llm-eval", "llm-evals", "llm-evaluation-framework", "mcp", "openrouter", "optimizer", "sdk", "tool-calling" ], "category": "Agent Eval", "momentum": 27, "rank": 232 }, { "name": "-L-", "full_name": "jettbrains/-L-", "slug": "jettbrains-l", "url": "https://github.com/jettbrains/-L-", "owner": "jettbrains", "owner_avatar": "https://avatars.githubusercontent.com/u/57115262?v=4", "stars": 169, "forks": 48, "open_issues": 15, "language": "", "license": "GPL-3.0", "pushed_at": "2021-08-18T13:41:54Z", "created_at": "2019-10-28T21:50:26Z", "description": "W3C Strategic Highlights September 2019 This report was prepared for the September 2019 W3C Advisory Committee Meeting (W3C Member link). See the accompanying W3C Fact Sheet — September 2019. For the previous edition, see the April 2019 W3C Strategic Highlights. For future editions of this report, please consult the latest version. A Chinese translation is available. ☰ Contents Introduction Future Web Standards Meeting Industry Needs Web Payments Digital Publishing Media and Entertainment Web & Telecommunications Real-Time Communications (WebRTC) Web & Networks Automotive Web of Things Strengthening the Core of the Web HTML CSS Fonts SVG Audio Performance Web Performance WebAssembly Testing Browser Testing and Tools WebPlatform Tests Web of Data Web for All Security, Privacy, Identity Internationalization (i18n) Web Accessibility Outreach to the world W3C Developer Relations W3C Training Translations W3C Liaisons Introduction This report highlights recent work of enhancement of the existing landscape of the Web platform and innovation for the growth and strength of the Web. 33 working groups and a dozen interest groups enable W3C to pursue its mission through the creation of Web standards, guidelines, and supporting materials. We track the tremendous work done across the Consortium through homogeneous work-spaces in Github which enables better monitoring and management. We are in the middle of a period where we are chartering numerous working groups which demonstrate the rapid degree of change for the Web platform: After 4 years, we are nearly ready to publish a Payment Request API Proposed Recommendation and we need to soon charter follow-on work. In the last year we chartered the Web Payment Security Interest Group. In the last year we chartered the Web Media Working Group with 7 specifications for next generation Media support on the Web. We have Accessibility Guidelines under W3C Member review which includes Silver, a new approach. We have just launched the Decentralized Identifier Working Group which has tremendous potential because Decentralized Identifier (DID) is an identifier that is globally unique, resolveable with high availability, and cryptographically verifiable. We have Privacy IG (PING) under W3C Member review which strengthens our focus on the tradeoff between privacy and function. We have a new CSS charter under W3C Member review which maps the group's work for the next three years. In this period, W3C and the WHATWG have succesfully completed the negotiation of a Memorandum of Understanding rooted in the mutual belief that that having two distinct specifications claiming to be normative is generally harmful for the Web community. The MOU, signed last May, describes how the two organizations are to collaborate on the development of a single authoritative version of the HTML and DOM specifications. W3C subsequently rechartered the HTML Working Group to assist the W3C community in raising issues and proposing solutions for the HTML and DOM specifications, and for the production of W3C Recommendations from WHATWG Review Drafts. As the Web evolves continuously, some groups are looking for ways for specifications to do so as well. So-called \"evergreen recommendations\" or \"living standards\" aim to track continuous development (and maintenance) of features, on a feature-by-feature basis, while getting review and patent commitments. We see the maturation and further development of an incredible number of new technologies coming to the Web. Continued progress in many areas demonstrates the vitality of the W3C and the Web community, as the rest of the report illustrates. Future Web Standards W3C has a variety of mechanisms for listening to what the community thinks could become good future Web standards. These include discussions with the Membership, discussions with other standards bodies, the activities of thousands of participants in over 300 community groups, and W3C Workshops. There are lots of good ideas. The W3C strategy team has been identifying promising topics and invites public participation. Future, recent and under consideration Workshops include: Inclusive XR (5-6 November 2019, Seattle, WA, USA) to explore existing and future approaches on making Virtual and Augmented Reality experiences more inclusive, including to people with disabilities; W3C Workshop on Data Models for Transportation (12-13 September 2019, Palo Alto, CA, USA) W3C Workshop on Web Games (27-28 June 2019, Redmond, WA, USA), view report Second W3C Workshop on the Web of Things (3-5 June 2019, Munich, Germany) W3C Workshop on Web Standardization for Graph Data; Creating Bridges: RDF, Property Graph and SQL (4-6 March 2019, Berlin, Germany), view report Web & Machine Learning. The Strategy Funnel documents the staff's exploration of potential new work at various phases: Exploration and Investigation, Incubation and Evaluation, and eventually to the chartering of a new standards group. The Funnel view is a GitHub Project where new area are issues represented by “cards” which move through the columns, usually from left to right. Most cards start in Exploration and move towards Chartering, or move out of the funnel. Public input is welcome at any stage but particularly once Incubation has begun. This helps W3C identify work that is sufficiently incubated to warrant standardization, to review the ecosystem around the work and indicate interest in participating in its standardization, and then to draft a charter that reflects an appropriate scope. Ongoing feedback can speed up the overall standardization process. Since the previous highlights document, W3C has chartered a number of groups, and started discussion on many more: Newly Chartered or Rechartered Web Application Security WG (03-Apr) Web Payment Security IG (17-Apr) Patent and Standards IG (24-Apr) Web Applications WG (14-May) Web & Networks IG (16-May) Media WG (23-May) Media and Entertainment IG (06-Jun) HTML WG (06-Jun) Decentralized Identifier WG (05-Sep) Extended Privacy IG (PING) (30-Sep) Verifiable Claims WG (30-Sep) Service Workers WG (31-Dec) Dataset Exchange WG (31-Dec) Web of Things Working Group (31-Dec) Web Audio Working Group (31-Dec) Proposed charters / Advance Notice Accessibility Guidelines WG Privacy IG (PING) RDF Literal Direction WG Timed Text WG CSS WG Web Authentication WG Closed Internationalization Tag Set IG Meeting Industry Needs Web Payments All Web Payments specifications W3C's payments standards enable a streamlined checkout experience, enabling a consistent user experience across the Web with lower front end development costs for merchants. Users can store and reuse information and more quickly and accurately complete online transactions. The Web Payments Working Group has republished Payment Request API as a Candidate Recommendation, aiming to publish a Proposed Recommendation in the Fall 2019, and is discussing use cases and features for Payment Request after publication of the 1.0 Recommendation. Browser vendors have been finalizing implementation of features added in the past year (view the implementation report). As work continues on the Payment Handler API and its implementation (currently in Chrome and Edge Canary), one focus in 2019 is to increase adoption in other browsers. Recently, Mastercard demonstrated the use of Payment Request API to carry out EMVCo's Secure Remote Commerce (SRC) protocol whose payment method definition is being developed with active participation by Visa, Mastercard, American Express, and Discover. Payment method availability is a key factor in merchant considerations about adopting Payment Request API. The ability to get uniform adoption of a new payment method such as Secure Remote Commerce (SRC) also depends on the availability of the Payment Handler API in browsers, or of proprietary alternatives. Web Monetization, which the Web Payments Working Group will discuss again at its face-to-face meeting in September, can be used to enable micropayments as an alternative revenue stream to advertising. Since the beginning of 2019, Amazon, Brave Software, JCB, Certus Cybersecurity Solutions and Netflix have joined the Web Payments Working Group. In April, W3C launched the Web Payment Security Group to enable W3C, EMVCo, and the FIDO Alliance to collaborate on a vision for Web payment security and interoperability. Participants will define areas of collaboration and identify gaps between existing technical specifications in order to increase compatibility among different technologies, such as: How do SRC, FIDO, and Payment Request relate? The Payment Services Directive 2 (PSD2) regulations in Europe are scheduled to take effect in September 2019. What is the role of EMVCo, W3C, and FIDO technologies, and what is the current state of readiness for the deadline? How can we improve privacy on the Web at the same time as we meet industry requirements regarding user identity? Digital Publishing All Digital Publishing specifications, Publication milestones The Web is the universal publishing platform. Publishing is increasingly impacted by the Web, and the Web increasingly impacts Publishing. Topic of particular interest to Publishing@W3C include typography and layout, accessibility, usability, portability, distribution, archiving, offline access, print on demand, and reliable cross referencing. And the diverse publishing community represented in the groups consist of the traditional \"trade\" publishers, ebook reading system manufacturers, but also publishers of audio book, scholarly journals or educational materials, library scientists or browser developers. The Publishing Working Group currently concentrates on Audiobooks which lack a comprehensive standard, thus incurring extra costs and time to publish in this booming market. Active development is ongoing on the future standard: Publication Manifest Audiobook profile for Web Publications Lightweight Packaging Format The BD Comics Manga Community Group, the Synchronized Multimedia for Publications Community Group, the Publishing Community Group and a future group on archival, are companions to the working group where specific work is developed and incubated. The Publishing Community Group is a recently launched incubation channel for Publishing@W3C. The goal of the group is to propose, document, and prototype features broadly related to: publications on the Web reading modes and systems and the user experience of publications The EPUB 3 Community Group has successfully completed the revision of EPUB 3.2. The Publishing Business Group fosters ongoing participation by members of the publishing industry and the overall ecosystem in the development of Web infrastructure to better support the needs of the industry. The Business Group serves as an additional conduit to the Publishing Working Group and several Community Groups for feedback between the publishing ecosystem and W3C. The Publishing BG has played a vital role in fostering and advancing the adoption and continued development of EPUB 3. In particular the BG provided critical support to the update of EPUBCheck to validate EPUB content to the new EPUB 3.2 specification. This resulted in the development, in conjunction with the EPUB3 Community Group, of a new generation of EPUBCheck, i.e., EPUBCheck 4.2 production-ready release. Media and Entertainment All Media specifications The Media and Entertainment vertical tracks media-related topics and features that create immersive experiences for end users. HTML5 brought standard audio and video elements to the Web. Standardization activities since then have aimed at turning the Web into a professional platform fully suitable for the delivery of media content and associated materials, enabling missing features to stream video content on the Web such as adaptive streaming and content protection. Together with Microsoft, Comcast, Netflix and Google, W3C received an Technology & Engineering Emmy Award in April 2019 for standardization of a full TV experience on the Web. Current goals are to: Reinforce core media technologies: Creation of the Media Working Group, to develop media-related specifications incubated in the WICG (e.g. Media Capabilities, Picture-in-picture, Media Session) and maintain maintain/evolve Media Source Extensions (MSE) and Encrypted Media Extensions (EME). Improve support for Media Timed Events: data cues incubation. Enhance color support (HDR, wide gamut), in scope of the CSS WG and in the Color on the Web CG. Reduce fragmentation: Continue annual releases of a common and testable baseline media devices, in scope of the Web Media APIs CG and in collaboration with the CTA WAVE Project. Maintain the Road-map of Media Technologies for the Web which highlights Web technologies that can be used to build media applications and services, as well as known gaps to enable additional use cases. Create the future: Discuss perspectives for Media and Entertainment for the Web. Bring the power of GPUs to the Web (graphics, machine learning, heavy processing), under incubation in the GPU for the Web CG. Transition to a Working Group is under discussion. Determine next steps after the successful W3C Workshop on Web Games of June 2019. View the report. Timed Text The Timed Text Working Group develops and maintains formats used for the representation of text synchronized with other timed media, like audio and video, and notably works on TTML, profiles of TTML, and WebVTT. Recent progress includes: A robust WebVTT implementation report poises the specification for publication as a proposed recommendation. Discussions around re-chartering, notably to add a TTML Profile for Audio Description deliverable to the scope of the group, and clarify that rendering of captions within XR content is also in scope. Immersive Web Hardware that enables Virtual Reality (VR) and Augmented Reality (AR) applications are now broadly available to consumers, offering an immersive computing platform with both new opportunities and challenges. The ability to interact directly with immersive hardware is critical to ensuring that the web is well equipped to operate as a first-class citizen in this environment. The Immersive Web Working Group has been stabilizing the WebXR Device API while the companion Immersive Web Community Group incubates the next series of features identified as key for the future of the Immersive Web. W3C plans a workshop focused on the needs and benefits at the intersection of VR & Accessibility (Inclusive XR), on 5-6 November 2019 in Seattle, WA, USA, to explore existing and future approaches on making Virtual and Augmented Reality experiences more inclusive. Web & Telecommunications The Web is the Open Platform for Mobile. Telecommunication service providers and network equipment providers have long been critical actors in the deployment of Web technologies. As the Web platform matures, it brings richer and richer capabilities to extend existing services to new users and devices, and propose new and innovative services. Real-Time Communications (WebRTC) All Real-Time Communications specifications WebRTC has reshaped the whole communication landscape by making any connected device a potential communication end-point, bringing audio and video communications anywhere, on any network, vastly expanding the ability of operators to reach their customers. WebRTC serves as the corner-stone of many online communication and collaboration services. The WebRTC Working Group aims to bringing WebRTC 1.0 (and companion specification Media Capture and Streams) to Recommendation by the end of 2019. Intense efforts are focused on testing (supported by a dedicated hackathon at IETF 104) and interoperability. The group is considering pushing features that have not gotten enough traction to separate modules or to a later minor revision of the spec. Beyond WebRTC 1.0, the WebRTC Working Group will focus its efforts on WebRTC NV which the group has started documenting by identifying use cases. Web & Networks Recently launched, in the wake of the May 2018 Web5G workshop, the Web & Networks Interest Group is chaired by representatives from AT&T, China Mobile and Intel, with a goal to explore solutions for web applications to achieve better performance and resource allocation, both on the device and network. The group's first efforts are around use cases, privacy & security requirements and liaisons. Automotive All Automotive specifications To create a rich application ecosystem for vehicles and other devices allowed to connect to the vehicle, the W3C Automotive Working Group is delivering a service specification to expose all common vehicle signals (engine temperature, fuel/charge level, range, tire pressure, speed, etc.) The Vehicle Information Service Specification (VISS), which is a Candidate Recommendation, is seeing more implementations across the industry. It provides the access method to a common data model for all the vehicle signals –presently encapsulating a thousand or so different data elements– and will be growing to accommodate the advances in automotive such as autonomous and driver assist technologies and electrification. The group is already working on a successor to VISS, leveraging the underlying data model and the VIWI submission from Volkswagen, for a more robust means of accessing vehicle signals information and the same paradigm for other automotive needs including location-based services, media, notifications and caching content. The Automotive and Web Platform Business Group acts as an incubator for prospective standards work. One of its task forces is using W3C VISS in performing data sampling and off-boarding the information to the cloud. Access to the wealth of information that W3C's auto signals standard exposes is of interest to regulators, urban planners, insurance companies, auto manufacturers, fleet managers and owners, service providers and others. In addition to components needed for data sampling and edge computing, capturing user and owner consent, information collection methods and handling of data are in scope. The upcoming W3C Workshop on Data Models for Transportation (September 2019) is expected to focus on the need of additional ontologies around transportation space. Web of Things All Web of Things specifications W3C's Web of Things work is designed to bridge disparate technology stacks to allow devices to work together and achieve scale, thus enabling the potential of the Internet of Things by eliminating fragmentation and fostering interoperability. Thing descriptions expressed in JSON-LD cover the behavior, interaction affordances, data schema, security configuration, and protocol bindings. The Web of Things complements existing IoT ecosystems to reduce the cost and risk for suppliers and consumers of applications that create value by combining multiple devices and information services. There are many sectors that will benefit, e.g. smart homes, smart cities, smart industry, smart agriculture, smart healthcare and many more. The Web of Things Working Group is finishing the initial Web of Things standards, with support from the Web of Things Interest Group: Web of Things Architecture Thing Descriptions Strengthening the Core of the Web HTML The HTML Working Group was chartered early June to assist the W3C community in raising issues and proposing solutions for the HTML and DOM specifications, and to produce W3C Recommendations from WHATWG Review Drafts. A few days before, W3C and the WHATWG signed a Memorandum of Understanding outlining the agreement to collaborate on the development of a single version of the HTML and DOM specifications. Issues and proposed solutions for HTML and DOM done via the newly rechartered HTML Working Group in the WHATWG repositories The HTML Working Group is targetting November 2019 to bring HTML and DOM to Candidate Recommendations. CSS All CSS specifications CSS is a critical part of the Open Web Platform. The CSS Working Group gathers requirements from two large groups of CSS users: the publishing industry and application developers. Within W3C, those groups are exemplified by the Publishing groups and the Web Platform Working Group. The former requires things like better pagination support and advanced font handling, the latter needs intelligent (and fast!) scrolling and animations. What we know as CSS is actually a collection of almost a hundred specifications, referred to as ‘modules’. The current state of CSS is defined by a snapshot, updated once a year. The group also publishes an index defining every term defined by CSS specifications. Fonts All Fonts specifications The Web Fonts Working Group develops specifications that allow the interoperable deployment of downloadable fonts on the Web, with a focus on Progressive Font Enrichment as well as maintenance of WOFF Recommendations. Recent and ongoing work includes: Early API experiments by Adobe and Monotype have demonstrated the feasibility of a font enrichment API, where a server delivers a font with minimal glyph repertoire and the client can query the full repertoire and request additional subsets on-the-fly. In other experiments, the Brotli compression used in WOFF 2 was extended to support shared dictionaries and patch update. Metrics to quantify improvement are a current hot discussion topic. The group will meet at ATypi 2019 in Japan, to gather requirements from the international typography community. The group will first produce a report summarizing the strengths and weaknesses of each prototype solution by Q2 2020. SVG All SVG specifications SVG is an important and widely-used part of the Open Web Platform. The SVG Working Group focuses on aligning the SVG 2.0 specification with browser implementations, having split the specification into a currently-implemented 2.0 and a forward-looking 2.1. Current activity is on stabilization, increased integration with the Open Web Platform, and test coverage analysis. The Working Group was rechartered in March 2019. A new work item concerns native (non-Web-browser) uses of SVG as a non-interactive, vector graphics format. Audio The Web Audio Working Group was extended to finish its work on the Web Audio API, expecting to publish it as a Recommendation by year end. The specification enables synthesizing audio in the browser. Audio operations are performed with audio nodes, which are linked together to form a modular audio routing graph. Multiple sources — with different types of channel layout — are supported. This modular design provides the flexibility to create complex audio functions with dynamic effects. The first version of Web Audio API is now feature complete and is implemented in all modern browsers. Work has started on the next version, and new features are being incubated in the Audio Community Group. Performance Web Performance All Web Performance specifications There are currently 18 specifications in development in the Web Performance Working Group aiming to provide methods to observe and improve aspects of application performance of user agent features and APIs. The W3C team is looking at related work incubated in the W3C GPU for the Web (WebGPU) Community Group which is poised to transition to a W3C Working Group. A preliminary draft charter is available. WebAssembly All WebAssembly specifications WebAssembly improves Web performance and power by being a virtual machine and execution environment enabling loaded pages to run native (compiled) code. It is deployed in Firefox, Edge, Safari and Chrome. The specification will soon reach Candidate Recommendation. WebAssembly enables near-native performance, optimized load time, and perhaps most importantly, a compilation target for existing code bases. While it has a small number of native types, much of the performance increase relative to Javascript derives from its use of consistent typing. WebAssembly leverages decades of optimization for compiled languages and the byte code is optimized for compactness and streaming (the web page starts executing while the rest of the code downloads). Network and API access all occurs through accompanying Javascript libraries -- the security model is identical to that of Javascript. Requirements gathering and language development occur in the Community Group while the Working Group manages test development, community review and progression of specifications on the Recommendation Track. Testing Browser testing plays a critical role in the growth of the Web by: Improving the reliability of Web technology definitions; Improving the quality of implementations of these technologies by helping vendors to detect bugs in their products; Improving the data available to Web developers on known bugs and deficiencies of Web technologies by publishing results of these tests. Browser Testing and Tools The Browser Testing and Tools Working Group is developing WebDriver version 2, having published last year the W3C Recommendation of WebDriver. WebDriver acts as a remote control interface that enables introspection and control of user agents, provides a platform- and language-neutral wire protocol as a way for out-of-process programs to remotely instruct the behavior of Web, and emulates the actions of a real person using the browser. WebPlatform Tests The WebPlatform Tests project now provides a mechanism which allows to fully automate tests that previously needed to be run manually: TestDriver. TestDriver enables sending trusted key and mouse events, sending complex series of trusted pointer and key interactions for things like in-content drag-and-drop or pinch zoom, and even file upload. Since 2014 W3C began work on this coordinated open-source effort to build a cross-browser test suite for the Web Platform, which WHATWG, and all major browsers adopted. Web of Data All Data specifications There have been several great success stories around the standardization of data on the web over the past year. Verifiable Claims seems to have significant uptake. It is also significant that the Distributed Identifier WG charter has received numerous favorable reviews, and was just recently launched. JSON-LD has been a major success with the large deployment on Web sites via schema.org. JSON-LD 1.1 completed technical work, about to transition to CR More than 25% of websites today include schema.org data in JSON-LD The Web of Things description is in CR since May, making use of JSON-LD Verifiable Credentials data model is in CR since July, also making use of JSON-LD Continued strong interest in decentralized identifiers Engagement from the TAG with reframing core documents, such as Ethical Web Principles, to include data on the web within their scope Data is increasingly important for all organizations, especially with the rise of IoT and Big Data. W3C has a mature and extensive suite of standards relating to data that were developed over two decades of experience, with plans for further work on making it easier for developers to work with graph data and knowledge graphs. Linked Data is about the use of URIs as names for things, the ability to dereference these URIs to get further information and to include links to other data. There are ever-increasing sources of open Linked Data on the Web, as well as data services that are restricted to the suppliers and consumers of those services. The digital transformation of industry is seeking to exploit advanced digital technologies. This will facilitate businesses to integrate horizontally along the supply and value chains, and vertically from the factory floor to the office floor. W3C is seeking to make it easier to support enterprise-wide data management and governance, reflecting the strategic importance of data to modern businesses. Traditional approaches to data have focused on tabular databases (SQL/RDBMS), Comma Separated Value (CSV) files, and data embedded in PDF documents and spreadsheets. We're now in midst of a major shift to graph data with nodes and labeled directed links between them. Graph data is: Faster than using SQL and associated JOIN operations More favorable to integrating data from heterogeneous sources Better suited to situations where the data model is evolving In the wake of the recent W3C Workshop on Graph Data we are in the process of launching a Graph Standardization Business Group to provide a business perspective with use cases and requirements, to coordinate technical standards work and liaisons with external organizations. Web for All Security, Privacy, Identity All Security specifications, all Privacy specifications Authentication on the Web As the WebAuthn Level 1 W3C Recommendation published last March is seeing wide implementation and adoption of strong cryptographic authentication, work is proceeding on Level 2. The open standard Web API gives native authentication technology built into native platforms, browsers, operating systems (including mobile) and hardware, offering protection against hacking, credential theft, phishing attacks, thus aiming to end the era of passwords as a security construct. You may read more in our March press release. Privacy An increasing number of W3C specifications are benefitting from Privacy and Security review; there are security and privacy aspects to every specification. Early review is essential. Working with the TAG, the Privacy Interest Group has updated the Self-Review Questionnaire: Security and Privacy. Other recent work of the group includes public blogging further to the exploration of anti-patterns in standards and permission prompts. Security The Web Application Security Working Group adopted Feature Policy, aiming to allow developers to selectively enable, disable, or modify the behavior of some of these browser features and APIs within their application; and Fetch Metadata, aiming to provide servers with enough information to make a priori decisions about whether or not to service a request based on the way it was made, and the context in which it will be used. The Web Payment Security Interest Group, launched last April, convenes members from W3C, EMVCo, and the FIDO Alliance to discuss cooperative work to enhance the security and interoperability of Web payments (read more about payments). Internationalization (i18n) All Internationalization specifications, educational articles related to Internationalization, spec developers checklist Only a quarter or so current Web users use English online and that proportion will continue to decrease as the Web reaches more and more communities of limited English proficiency. If the Web is to live up to the \"World Wide\" portion of its name, and for the Web to truly work for stakeholders all around the world engaging with content in various languages, it must support the needs of worldwide users as they engage with content in the various languages. The growth of epublishing also brings requirements for new features and improved typography on the Web. It is important to ensure the needs of local communities are captured. The W3C Internationalization Initiative was set up to increase in-house resources dedicated to accelerating progress in making the World Wide Web \"worldwide\" by gathering user requirements, supporting developers, and education & outreach. For an overview of current projects see the i18n radar. W3C's Internationalization efforts progressed on a number of fronts recently: Requirements: New African and European language groups will work on the gap analysis, errata and layout requirements. Gap analysis: Japanese, Devanagari, Bengali, Tamil, Lao, Khmer, Javanese, and Ethiopic updated in the gap-analysis documents. Layout requirements document: notable progress tracked in the Southeast Asian Task Force while work continues on Chinese layout requirements. Developer support: Spec reviews: the i18n WG continues active review of specifications of the WHATWG and other W3C Working Groups. Short review checklist: easy way to begin a self-review to help spec developers understand what aspects of their spec are likely to need attention for internationalization, and points them to more detailed checklists for the relevant topics. It also helps those reviewing specs for i18n issues. Strings on the Web: Language and Direction Metadata lays out issues and discusses potential solutions for passing information about language and direction with strings in JSON or other data formats. The document was rewritten for clarity, and expanded. The group is collaborating with the JSON-LD and Web Publishing groups to develop a plan for updating RDF, JSON-LD and related specifications to handle metadata for base direction of text (bidi). User-friendly test format: a new format was developed for Internationalization Test Suite tests, which displays helpful information about how the test works. This particularly useful because those tests are pointed to by educational materials and gap-analysis documents. Web Platform Tests: a large number of tests in the i18n test suite have been ported to the WPT repository, including: css-counter-styles, css-ruby, css-syntax, css-test, css-text-decor, css-writing-modes, and css-pseudo. Education & outreach: (for all educational materials, see the HTML & CSS Authoring Techniques) Web Accessibility All Accessibility specifications, WAI resources The Web Accessibility Initiative supports W3C's Web for All mission. Recent achievements include: Education and training: Inaccessibility of CAPTCHA updated to bring our analysis and recommendations up to date with CAPTCHA practice today, concluding two years of extensive work and invaluable input from the public (read more on the W3C Blog Learn why your web content and applications should be accessible. The Education and Outreach Working Group has completed revision and updating of the Business Case for Digital Accessibility. Accessibility guidelines: The Accessibility Guidelines Working Group has continued to update WCAG Techniques and Understanding WCAG 2.1; and published a Candidate Recommendation of Accessibility Conformance Testing Rules Format 1.0 to improve inter-rater reliability when evaluating conformance of web content to WCAG An updated charter is being developed to host work on \"Silver\", the next generation accessibility guidelines (WCAG 2.2) There are accessibility aspects to most specifications. Check your work with the FAST checklist. Outreach to the world W3C Developer Relations To foster the excellent feedback loop between Web Standards development and Web developers, and to grow participation from that diverse community, recent W3C Developer Relations activities include: @w3cdevs tracks the enormous amount of work happening across W3C W3C Track during the Web Conference 2019 in San Francisco Tech videos: W3C published the 2019 Web Games Workshop videos The 16 September 2019 Developer Meetup in Fukuoka, Japan, is open to all and will combine a set of technical demos prepared by W3C groups, and a series of talks on a selected set of W3C technologies and projects W3C is involved with Mozilla, Google, Samsung, Microsoft and Bocoup in the organization of ViewSource 2019 in Amsterdam (read more on the W3C Blog) W3C Training In partnership with EdX, W3C's MOOC training program, W3Cx offers a complete \"Front-End Web Developer\" (FEWD) professional certificate program that consists of a suite of five courses on the foundational languages that power the Web: HTML5, CSS and JavaScript. We count nearly 900K students from all over the world. Translations Many Web users rely on translations of documents developed at W3C whose official language is English. W3C is extremely grateful to the continuous efforts of its community in ensuring our various deliverables in general, and in our specifications in particular, are made available in other languages, for free, ensuring their exposure to a much more diverse set of readers. Last Spring we developed a more robust system, a new listing of translations of W3C specifications and updated the instructions on how to contribute to our translation efforts. W3C Liaisons Liaisons and coordination with numerous organizations and Standards Development Organizations (SDOs) is crucial for W3C to: make sure standards are interoperable coordinate our respective agenda in Internet governance: W3C participates in ICANN, GIPO, IGF, the I* organizations (ICANN, IETF, ISOC, IAB). ensure at the government liaison level that our standards work is officially recognized when important to our membership so that products based on them (often done by our members) are part of procurement orders. W3C has ARO/PAS status with ISO. W3C participates in the EU MSP and Rolling Plan on Standardization ensure the global set of Web and Internet standards form a compatible stack of technologies, at the technical and policy level (patent regime, fragmentation, use in policy making) promote Standards adoption equally by the industry, the public sector, and the public at large Coralie Mercier, Editor, W3C Marketing & Communications $Id: Overview.html,v 1.60 2019/10/15 12:05:52 coralie Exp $ Copyright © 2019 W3C ® (MIT, ERCIM, Keio, Beihang) Usage policies apply.", "topics": [], "category": "Observability", "momentum": 26, "rank": 233 }, { "name": "example-rag-app", "full_name": "ajac-zero/example-rag-app", "slug": "ajac-zero-example-rag-app", "url": "https://github.com/ajac-zero/example-rag-app", "owner": "ajac-zero", "owner_avatar": "https://avatars.githubusercontent.com/u/115488250?v=4", "stars": 165, "forks": 26, "open_issues": 1, "language": "TypeScript", "license": "MIT", "pushed_at": "2026-01-15T20:18:12Z", "created_at": "2025-02-18T16:15:24Z", "description": "Open-Source RAG app with LLM Observability (Langfuse), support for 100+ providers (LiteLLM), Dockerized, Full Type-checking, 100% Test coverage, and more...", "topics": [ "ai", "llm", "rag" ], "category": "Observability", "momentum": 26, "rank": 234 }, { "name": "LLMSecurityGuide", "full_name": "requie/LLMSecurityGuide", "slug": "requie-llmsecurityguide", "url": "https://github.com/requie/LLMSecurityGuide", "owner": "requie", "owner_avatar": "https://avatars.githubusercontent.com/u/8768293?v=4", "stars": 161, "forks": 21, "open_issues": 2, "language": "", "license": "", "pushed_at": "2026-04-03T12:51:21Z", "created_at": "2025-10-07T23:18:36Z", "description": "A comprehensive reference for securing Large Language Models (LLMs). Covers OWASP GenAI Top-10 risks, prompt injection, adversarial attacks, real-world incidents, and practical defenses. Includes catalogs of red-teaming tools, guardrails, and mitigation strategies to help developers, researchers, and security teams deploy AI responsibly.", "topics": [ "ai-safety", "ai-security", "ai-security-tool", "generative-ai-security", "generative-ai-security-assurance", "llm-security", "llm-security-compliance-prompt-injection", "llm-vulnerabilities", "offensive-security", "prompt-injection", "prompt-injection-defense", "prompt-injection-llm-security", "red-teaming" ], "category": "Red Teaming & Safety", "momentum": 26, "rank": 235 }, { "name": "llemonstack", "full_name": "LLemonStack/llemonstack", "slug": "llemonstack-llemonstack", "url": "https://github.com/LLemonStack/llemonstack", "owner": "LLemonStack", "owner_avatar": "https://avatars.githubusercontent.com/u/201296664?v=4", "stars": 147, "forks": 28, "open_issues": 2, "language": "TypeScript", "license": "AGPL-3.0", "pushed_at": "2026-02-03T22:31:32Z", "created_at": "2025-03-03T21:16:33Z", "description": "All-in-one local low-code AI agent development platform. Installs and runs n8n, Flowise, Browser-Use, Qdrant, Ollama, and more. Proxies LLM requests through LiteLLM with Langfuse for observability.", "topics": [], "category": "Observability", "momentum": 26, "rank": 236 }, { "name": "Awesome-LLM-Red-Teaming", "full_name": "user1342/Awesome-LLM-Red-Teaming", "slug": "user1342-awesome-llm-red-teaming", "url": "https://github.com/user1342/Awesome-LLM-Red-Teaming", "owner": "user1342", "owner_avatar": "https://avatars.githubusercontent.com/u/28961606?v=4", "stars": 147, "forks": 27, "open_issues": 11, "language": "", "license": "MIT", "pushed_at": "2025-09-04T17:40:25Z", "created_at": "2024-11-11T07:45:16Z", "description": "A curated list of awesome LLM Red Teaming training, resources, and tools.", "topics": [ "ai", "artificial-intelligence", "computer-security", "large-language-models", "llm", "machine-learning", "ml", "penetration-testing", "red-team-tools", "red-teaming", "reverse-engineering", "vulnerabilit-research", "vulnerability-assessment" ], "category": "Red Teaming & Safety", "momentum": 26, "rank": 237 }, { "name": "super_pe_evaluation", "full_name": "yance521/super_pe_evaluation", "slug": "yance521-super-pe-evaluation", "url": "https://github.com/yance521/super_pe_evaluation", "owner": "yance521", "owner_avatar": "https://avatars.githubusercontent.com/u/176866865?v=4", "stars": 17, "forks": 0, "open_issues": 0, "language": "Go", "license": "Apache-2.0", "pushed_at": "2026-07-09T12:51:43Z", "created_at": "2026-07-08T11:10:24Z", "description": "A TDD-style prompt evaluation skill — build reusable eval datasets, run prompt versions against locked test cases, score outputs with evidence, and compare results without overwriting history. Local-first, platform-agnostic, append-only.", "topics": [], "category": "Benchmarks", "momentum": 26, "rank": 238 }, { "name": "ollama-multirun", "full_name": "attogram/ollama-multirun", "slug": "attogram-ollama-multirun", "url": "https://github.com/attogram/ollama-multirun", "owner": "attogram", "owner_avatar": "https://avatars.githubusercontent.com/u/8653063?v=4", "stars": 16, "forks": 2, "open_issues": 1, "language": "Shell", "license": "MIT", "pushed_at": "2026-07-12T14:06:36Z", "created_at": "2025-05-30T15:38:52Z", "description": "Run a prompt against all, or some, of your models running on Ollama. Creates web pages with the output, performance statistics and model info. All in a single Bash shell script.", "topics": [ "ai", "ai-evaluation-tools", "attogram-project", "bash-script", "llm-eval", "llm-evaluation", "llm-evaluation-metrics", "ollama", "ollama-app", "ollama-interface", "static-site-generator" ], "category": "Eval Frameworks", "momentum": 26, "rank": 239 }, { "name": "sparse-frontier", "full_name": "PiotrNawrot/sparse-frontier", "slug": "piotrnawrot-sparse-frontier", "url": "https://github.com/PiotrNawrot/sparse-frontier", "owner": "PiotrNawrot", "owner_avatar": "https://avatars.githubusercontent.com/u/29411944?v=4", "stars": 134, "forks": 11, "open_issues": 3, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-01-27T17:08:42Z", "created_at": "2025-04-23T23:25:32Z", "description": "The evaluation framework for training-free sparse attention in LLMs", "topics": [], "category": "Eval Frameworks", "momentum": 25, "rank": 240 }, { "name": "llmkit", "full_name": "llmkit-ai/llmkit", "slug": "llmkit-ai-llmkit", "url": "https://github.com/llmkit-ai/llmkit", "owner": "llmkit-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/204102661?v=4", "stars": 120, "forks": 21, "open_issues": 11, "language": "Rust", "license": "MIT", "pushed_at": "2025-06-26T00:24:16Z", "created_at": "2025-01-21T02:19:54Z", "description": "A prompt management, versioning, testing, and evaluation inference server and UI toolkit. Provider agnostic and OpenAI API compatible.", "topics": [], "category": "Eval Frameworks", "momentum": 25, "rank": 241 }, { "name": "mobius-llm-adversity", "full_name": "Mobius-Dev/mobius-llm-adversity", "slug": "mobius-dev-mobius-llm-adversity", "url": "https://github.com/Mobius-Dev/mobius-llm-adversity", "owner": "Mobius-Dev", "owner_avatar": "https://avatars.githubusercontent.com/u/69324301?v=4", "stars": 120, "forks": 14, "open_issues": 0, "language": "", "license": "MIT", "pushed_at": "2025-08-15T08:31:27Z", "created_at": "2025-07-18T00:34:38Z", "description": "This repository documents a series of experiments focused on adversarial prompting and jailbreaks against large language models. It is part of my personal red teaming portfolio, intended to showcase prompt engineering techniques, jailbreak persistence, and alignment failure analysis.", "topics": [], "category": "Red Teaming & Safety", "momentum": 25, "rank": 242 }, { "name": "gandalf-llm-pentester", "full_name": "MrMoshkovitz/gandalf-llm-pentester", "slug": "mrmoshkovitz-gandalf-llm-pentester", "url": "https://github.com/MrMoshkovitz/gandalf-llm-pentester", "owner": "MrMoshkovitz", "owner_avatar": "https://avatars.githubusercontent.com/u/50079480?v=4", "stars": 117, "forks": 16, "open_issues": 0, "language": "Jupyter Notebook", "license": "NOASSERTION", "pushed_at": "2025-07-30T14:37:55Z", "created_at": "2025-07-30T12:50:13Z", "description": "Automated red-team toolkit for stress-testing LLM defences - Vector Attacks on LLMs (Gendalf Case Study)", "topics": [], "category": "Red Teaming & Safety", "momentum": 25, "rank": 243 }, { "name": "raga-llm-hub", "full_name": "raga-ai-hub/raga-llm-hub", "slug": "raga-ai-hub-raga-llm-hub", "url": "https://github.com/raga-ai-hub/raga-llm-hub", "owner": "raga-ai-hub", "owner_avatar": "https://avatars.githubusercontent.com/u/161833182?v=4", "stars": 115, "forks": 14, "open_issues": 2, "language": "Python", "license": "NOASSERTION", "pushed_at": "2024-09-09T10:53:31Z", "created_at": "2024-03-02T18:32:13Z", "description": "Framework for LLM evaluation, guardrails and security", "topics": [ "guardrails", "llm-evaluation", "llm-security", "llmops" ], "category": "Red Teaming & Safety", "momentum": 25, "rank": 244 }, { "name": "RAGEval", "full_name": "BytePioneer-AI/RAGEval", "slug": "bytepioneer-ai-rageval", "url": "https://github.com/BytePioneer-AI/RAGEval", "owner": "BytePioneer-AI", "owner_avatar": "https://avatars.githubusercontent.com/u/61337845?v=4", "stars": 114, "forks": 17, "open_issues": 11, "language": "TypeScript", "license": "", "pushed_at": "2026-04-16T07:40:52Z", "created_at": "2025-03-15T16:37:55Z", "description": "灵鉴 RAG评测系统 | RAGEval ✨ 开箱即用的RAG系统自动化评估工具 | One-stop RAG Evaluation Solution", "topics": [], "category": "RAG Eval", "momentum": 24, "rank": 245 }, { "name": "llm-jeopardy", "full_name": "aigoopy/llm-jeopardy", "slug": "aigoopy-llm-jeopardy", "url": "https://github.com/aigoopy/llm-jeopardy", "owner": "aigoopy", "owner_avatar": "https://avatars.githubusercontent.com/u/128152550?v=4", "stars": 109, "forks": 4, "open_issues": 3, "language": "JavaScript", "license": "MIT", "pushed_at": "2023-07-29T12:16:43Z", "created_at": "2023-04-24T12:52:51Z", "description": "Automated prompting and scoring framework to evaluate LLMs using updated human knowledge prompts", "topics": [], "category": "Eval Frameworks", "momentum": 24, "rank": 246 }, { "name": "autoarena", "full_name": "kolenaIO/autoarena", "slug": "kolenaio-autoarena", "url": "https://github.com/kolenaIO/autoarena", "owner": "kolenaIO", "owner_avatar": "https://avatars.githubusercontent.com/u/77010818?v=4", "stars": 108, "forks": 9, "open_issues": 4, "language": "TypeScript", "license": "Apache-2.0", "pushed_at": "2024-12-16T12:25:44Z", "created_at": "2024-08-28T19:55:28Z", "description": "Rank LLMs, RAG systems, and prompts using automated head-to-head evaluation", "topics": [ "ai", "evaluation", "hacktoberfest", "llm", "llm-evaluation", "rag", "testing" ], "category": "RAG Eval", "momentum": 24, "rank": 247 }, { "name": "web2code", "full_name": "MBZUAI-LLM/web2code", "slug": "mbzuai-llm-web2code", "url": "https://github.com/MBZUAI-LLM/web2code", "owner": "MBZUAI-LLM", "owner_avatar": "https://avatars.githubusercontent.com/u/131686674?v=4", "stars": 103, "forks": 8, "open_issues": 2, "language": "Python", "license": "", "pushed_at": "2024-10-23T12:13:24Z", "created_at": "2024-06-05T13:18:58Z", "description": "Web2Code: A Large-scale Webpage-to-Code Dataset and Evaluation Framework for Multimodal LLMs", "topics": [], "category": "Coding Eval", "momentum": 24, "rank": 248 }, { "name": "qdrant-rag-eval", "full_name": "qdrant/qdrant-rag-eval", "slug": "qdrant-qdrant-rag-eval", "url": "https://github.com/qdrant/qdrant-rag-eval", "owner": "qdrant", "owner_avatar": "https://avatars.githubusercontent.com/u/73504361?v=4", "stars": 92, "forks": 27, "open_issues": 1, "language": "Jupyter Notebook", "license": "", "pushed_at": "2025-04-25T08:26:11Z", "created_at": "2024-04-03T12:32:35Z", "description": "This repo is the central repo for all the RAG Evaluation reference material and partner workshop", "topics": [], "category": "RAG Eval", "momentum": 23, "rank": 249 }, { "name": "auto-rag-eval", "full_name": "amazon-science/auto-rag-eval", "slug": "amazon-science-auto-rag-eval", "url": "https://github.com/amazon-science/auto-rag-eval", "owner": "amazon-science", "owner_avatar": "https://avatars.githubusercontent.com/u/70298811?v=4", "stars": 92, "forks": 15, "open_issues": 3, "language": "Python", "license": "Apache-2.0", "pushed_at": "2024-06-13T20:50:39Z", "created_at": "2024-05-20T09:24:49Z", "description": "Code repo for the ICML 2024 paper \"Automated Evaluation of Retrieval-Augmented Language Models with Task-Specific Exam Generation\"", "topics": [ "evaluation", "genai", "llm", "machine-learning" ], "category": "RAG Eval", "momentum": 23, "rank": 250 }, { "name": "just-eval", "full_name": "Re-Align/just-eval", "slug": "re-align-just-eval", "url": "https://github.com/Re-Align/just-eval", "owner": "Re-Align", "owner_avatar": "https://avatars.githubusercontent.com/u/150300417?v=4", "stars": 90, "forks": 7, "open_issues": 2, "language": "Python", "license": "MIT", "pushed_at": "2024-01-29T23:45:40Z", "created_at": "2023-11-19T02:51:26Z", "description": "A simple GPT-based evaluation tool for multi-aspect, interpretable assessment of LLMs.", "topics": [ "evaluation", "gpt4", "llm", "llm-eval", "llm-evaluation", "llm-evaluation-toolkit" ], "category": "Eval Frameworks", "momentum": 23, "rank": 251 }, { "name": "LLM-judge-reporting", "full_name": "UW-Madison-Lee-Lab/LLM-judge-reporting", "slug": "uw-madison-lee-lab-llm-judge-reporting", "url": "https://github.com/UW-Madison-Lee-Lab/LLM-judge-reporting", "owner": "UW-Madison-Lee-Lab", "owner_avatar": "https://avatars.githubusercontent.com/u/94162930?v=4", "stars": 86, "forks": 4, "open_issues": 0, "language": "Jupyter Notebook", "license": "", "pushed_at": "2025-11-27T03:04:07Z", "created_at": "2025-11-26T18:26:18Z", "description": "A simple plug-in framework that corrects bias and computes confidence intervals in reporting LLM-as-a-judge evaluation, and an adaptive algorithm that efficiently allocates calibration samples to reduce uncertainty in estimates.", "topics": [], "category": "Eval Frameworks", "momentum": 23, "rank": 252 }, { "name": "parea-sdk-py", "full_name": "parea-ai/parea-sdk-py", "slug": "parea-ai-parea-sdk-py", "url": "https://github.com/parea-ai/parea-sdk-py", "owner": "parea-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/135163833?v=4", "stars": 82, "forks": 13, "open_issues": 59, "language": "Python", "license": "Apache-2.0", "pushed_at": "2025-02-13T14:48:37Z", "created_at": "2023-07-24T02:57:39Z", "description": "Python SDK for experimenting, testing, evaluating & monitoring LLM-powered applications - Parea AI (YC S23)", "topics": [ "generative-ai", "good-first-issue", "llm", "llm-eval", "llm-evaluation", "llm-evaluation-framework", "llm-evaluation-toolkit", "llm-tools", "llmops", "llms-benchmarking", "metrics", "prompt-engineering" ], "category": "Observability", "momentum": 23, "rank": 253 }, { "name": "circle-guard-bench", "full_name": "whitecircle/circle-guard-bench", "slug": "whitecircle-circle-guard-bench", "url": "https://github.com/whitecircle/circle-guard-bench", "owner": "whitecircle", "owner_avatar": "https://avatars.githubusercontent.com/u/197516060?v=4", "stars": 81, "forks": 8, "open_issues": 3, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-03-07T12:39:03Z", "created_at": "2025-04-24T11:03:59Z", "description": "First-of-its-kind AI benchmark for evaluating the protection capabilities of large language model (LLM) guard systems (guardrails and safeguards)", "topics": [ "ai", "benchmark", "benchmarking", "guardrail", "guardrails", "jailbreak", "large-language-model", "large-language-models", "llm", "llm-as-a-judge", "llm-eval", "llm-evaluation", "llm-jailbreaks", "llm-security", "safeguard" ], "category": "Red Teaming & Safety", "momentum": 23, "rank": 254 }, { "name": "RedTeam-Agent", "full_name": "ktol1/RedTeam-Agent", "slug": "ktol1-redteam-agent", "url": "https://github.com/ktol1/RedTeam-Agent", "owner": "ktol1", "owner_avatar": "https://avatars.githubusercontent.com/u/94578931?v=4", "stars": 79, "forks": 13, "open_issues": 0, "language": "C", "license": "", "pushed_at": "2026-04-24T02:38:36Z", "created_at": "2026-02-28T12:16:58Z", "description": "RedTeam-Agent: AI-Powered Autonomous Red Team Framework via Model Context Protocol. AI红队与内网渗透自动化框架，支持 gogo, fscan, httpx, nuclei, impacket, playwright 等 15+ 渗透工具，让 LLM 直接化身安全审计黑客。", "topics": [ "active-directory", "ai-agent", "ai-security", "bloodhound", "cybersecurity", "ethical-hacking", "fscan", "impacket", "lateral-movement", "llm-tools", "mcp", "mcp-server", "model-context-protocol", "nuclei", "offensive-security", "penetration-testing", "pentesting", "red-team", "redteam", "windows-security" ], "category": "Red Teaming & Safety", "momentum": 23, "rank": 255 }, { "name": "llm-eval-simple", "full_name": "grigio/llm-eval-simple", "slug": "grigio-llm-eval-simple", "url": "https://github.com/grigio/llm-eval-simple", "owner": "grigio", "owner_avatar": "https://avatars.githubusercontent.com/u/8074?v=4", "stars": 72, "forks": 3, "open_issues": 2, "language": "Python", "license": "MIT", "pushed_at": "2026-02-28T08:06:21Z", "created_at": "2025-09-02T19:40:56Z", "description": "llm-eval-simple is a simple LLM evaluation framework with intermediate actions and prompt pattern selection", "topics": [ "llm", "llm-eval", "llm-evaluation-benchmark" ], "category": "Benchmarks", "momentum": 22, "rank": 256 }, { "name": "ruby_llm-tribunal", "full_name": "Alqemist-labs/ruby_llm-tribunal", "slug": "alqemist-labs-ruby-llm-tribunal", "url": "https://github.com/Alqemist-labs/ruby_llm-tribunal", "owner": "Alqemist-labs", "owner_avatar": "https://avatars.githubusercontent.com/u/22685823?v=4", "stars": 71, "forks": 7, "open_issues": 2, "language": "Ruby", "license": "MIT", "pushed_at": "2026-04-09T08:46:03Z", "created_at": "2026-01-15T14:26:57Z", "description": "LLM evaluation framework for Ruby, powered by RubyLLM. Tribunal provides tools for evaluating and testing LLM outputs, detecting hallucinations, measuring response quality, and ensuring safety. Perfect for RAG systems, chatbots, and any LLM-powered application.", "topics": [], "category": "Red Teaming & Safety", "momentum": 22, "rank": 257 }, { "name": "CATArena", "full_name": "AGI-Eval-Official/CATArena", "slug": "agi-eval-official-catarena", "url": "https://github.com/AGI-Eval-Official/CATArena", "owner": "AGI-Eval-Official", "owner_avatar": "https://avatars.githubusercontent.com/u/216111902?v=4", "stars": 69, "forks": 11, "open_issues": 0, "language": "Python", "license": "", "pushed_at": "2025-12-25T12:57:13Z", "created_at": "2025-10-24T10:05:36Z", "description": "CATArena is an engineering-level tournament evaluation platform for Large Language Model-driven code agents (LLM-driven code agents), based on an iterative competitive peer learning framework.", "topics": [], "category": "Agent Eval", "momentum": 22, "rank": 258 }, { "name": "VisualWebBench", "full_name": "VisualWebBench/VisualWebBench", "slug": "visualwebbench-visualwebbench", "url": "https://github.com/VisualWebBench/VisualWebBench", "owner": "VisualWebBench", "owner_avatar": "https://avatars.githubusercontent.com/u/165917046?v=4", "stars": 68, "forks": 6, "open_issues": 7, "language": "Python", "license": "", "pushed_at": "2024-10-19T12:29:01Z", "created_at": "2024-04-02T13:42:24Z", "description": "Evaluation framework for paper \"VisualWebBench: How Far Have Multimodal LLMs Evolved in Web Page Understanding and Grounding?\"", "topics": [ "computer-vision", "deep-learning", "evaluation", "foundation-models", "large-language-models", "large-multimodal-models", "llm", "llms", "machine-learning", "mllm", "multimodal", "multimodal-deep-learning", "multimodal-large-language-models", "natural-language-processing", "question-answering", "visual-question-answering" ], "category": "Eval Frameworks", "momentum": 22, "rank": 259 }, { "name": "acceptance-bench", "full_name": "ellydee/acceptance-bench", "slug": "ellydee-acceptance-bench", "url": "https://github.com/ellydee/acceptance-bench", "owner": "ellydee", "owner_avatar": "https://avatars.githubusercontent.com/u/228110671?v=4", "stars": 68, "forks": 1, "open_issues": 0, "language": "Python", "license": "MIT", "pushed_at": "2026-04-13T23:29:15Z", "created_at": "2025-10-16T18:30:13Z", "description": "A robust LLM evaluation framework measuring acceptance vs refusal across difficulty levels. Features multi-prompt variation testing, temperature sweeping, and LLM-as-judge evaluation.", "topics": [], "category": "Eval Frameworks", "momentum": 22, "rank": 260 }, { "name": "Jailjudge", "full_name": "usail-hkust/Jailjudge", "slug": "usail-hkust-jailjudge", "url": "https://github.com/usail-hkust/Jailjudge", "owner": "usail-hkust", "owner_avatar": "https://avatars.githubusercontent.com/u/114993847?v=4", "stars": 67, "forks": 6, "open_issues": 2, "language": "Python", "license": "MIT", "pushed_at": "2024-12-13T11:47:18Z", "created_at": "2024-10-10T09:44:56Z", "description": "JAILJUDGE: A comprehensive evaluation benchmark which includes a wide range of risk scenarios with complex malicious prompts (e.g., synthetic, adversarial, in-the-wild, and multi-language scenarios, etc.) along with high-quality human- annotated test datasets.", "topics": [], "category": "Red Teaming & Safety", "momentum": 22, "rank": 261 }, { "name": "rulm-sbs2", "full_name": "kuk/rulm-sbs2", "slug": "kuk-rulm-sbs2", "url": "https://github.com/kuk/rulm-sbs2", "owner": "kuk", "owner_avatar": "https://avatars.githubusercontent.com/u/153776?v=4", "stars": 61, "forks": 2, "open_issues": 0, "language": "Jupyter Notebook", "license": "", "pushed_at": "2023-09-26T06:21:18Z", "created_at": "2023-08-23T07:33:00Z", "description": "Бенчмарк сравнивает русские аналоги ChatGPT: Saiga, YandexGPT, Gigachat", "topics": [ "llm-eval", "russian-specific" ], "category": "Eval Frameworks", "momentum": 21, "rank": 262 }, { "name": "EvoSynth", "full_name": "dongdongunique/EvoSynth", "slug": "dongdongunique-evosynth", "url": "https://github.com/dongdongunique/EvoSynth", "owner": "dongdongunique", "owner_avatar": "https://avatars.githubusercontent.com/u/89110975?v=4", "stars": 61, "forks": 10, "open_issues": 1, "language": "Python", "license": "", "pushed_at": "2026-02-04T05:54:13Z", "created_at": "2025-11-16T15:23:06Z", "description": "EvoSynth is a SOTA automated LLM red-teaming framework that evolves executable, code-level jailbreak attack methods using multi-agent synthesis, evolutionary search, and self-correction.", "topics": [ "ai-safety", "automated-red-teaming", "jailbreak", "llm-security", "sota" ], "category": "Red Teaming & Safety", "momentum": 21, "rank": 263 }, { "name": "Oversight", "full_name": "user1342/Oversight", "slug": "user1342-oversight", "url": "https://github.com/user1342/Oversight", "owner": "user1342", "owner_avatar": "https://avatars.githubusercontent.com/u/28961606?v=4", "stars": 55, "forks": 2, "open_issues": 1, "language": "Python", "license": "GPL-3.0", "pushed_at": "2024-11-09T07:02:14Z", "created_at": "2024-11-05T16:16:16Z", "description": "A Completely Modular LLM Reverse Engineering, Red Teaming, and Vulnerability Research Framework.", "topics": [ "ai", "artificial-intelligence", "large-language-models", "llm", "machine-learning", "ml", "red-teaming", "security", "vulnerability-scanners" ], "category": "Red Teaming & Safety", "momentum": 21, "rank": 264 }, { "name": "OpenSkillEval", "full_name": "ALEX-nlp/OpenSkillEval", "slug": "alex-nlp-openskilleval", "url": "https://github.com/ALEX-nlp/OpenSkillEval", "owner": "ALEX-nlp", "owner_avatar": "https://avatars.githubusercontent.com/u/206792812?v=4", "stars": 14, "forks": 0, "open_issues": 0, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-06-15T02:14:09Z", "created_at": "2026-05-29T03:06:01Z", "description": "OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents", "topics": [ "agent-evaluation", "ai-agents", "benchmark", "llm-eval", "skill-evaluation" ], "category": "Agent Eval", "momentum": 21, "rank": 265 }, { "name": "multinear", "full_name": "multinear/multinear", "slug": "multinear-multinear", "url": "https://github.com/multinear/multinear", "owner": "multinear", "owner_avatar": "https://avatars.githubusercontent.com/u/186153610?v=4", "stars": 45, "forks": 1, "open_issues": 3, "language": "Python", "license": "MIT", "pushed_at": "2025-09-02T11:16:27Z", "created_at": "2024-11-25T13:28:37Z", "description": "Develop reliable AI apps", "topics": [ "evaluation", "llm", "llm-eval", "llm-evaluation", "llm-evaluation-framework", "llms", "llms-benchmarking", "reliability" ], "category": "Benchmarks", "momentum": 20, "rank": 266 }, { "name": "prompto", "full_name": "alan-turing-institute/prompto", "slug": "alan-turing-institute-prompto", "url": "https://github.com/alan-turing-institute/prompto", "owner": "alan-turing-institute", "owner_avatar": "https://avatars.githubusercontent.com/u/18304793?v=4", "stars": 38, "forks": 3, "open_issues": 22, "language": "Python", "license": "MIT", "pushed_at": "2025-07-18T11:50:10Z", "created_at": "2024-04-03T13:56:06Z", "description": "An open source library for asynchronous querying of LLM endpoints", "topics": [ "deep-learning", "hut23", "large-language-models", "llm-eval", "llm-evaluation", "llms", "machine-learning", "natural-language-processing", "nlp", "python", "transformer", "transformers" ], "category": "Eval Frameworks", "momentum": 19, "rank": 267 }, { "name": "dbt-llm-evals", "full_name": "paradime-io/dbt-llm-evals", "slug": "paradime-io-dbt-llm-evals", "url": "https://github.com/paradime-io/dbt-llm-evals", "owner": "paradime-io", "owner_avatar": "https://avatars.githubusercontent.com/u/66121117?v=4", "stars": 35, "forks": 5, "open_issues": 13, "language": "Python", "license": "NOASSERTION", "pushed_at": "2026-02-10T21:32:55Z", "created_at": "2026-01-19T16:01:18Z", "description": "The warehouse-native LLM evaluation package for dbt™ - monitor AI quality without data egress", "topics": [ "ai", "anthropic", "bigquery", "databricks", "dbt", "dbt-core", "dbt-packages", "gemini", "llm", "llm-eval", "llm-evaluation", "openai", "snowflake", "vertex-ai" ], "category": "Observability", "momentum": 18, "rank": 268 }, { "name": "vibraniumdome", "full_name": "genia-dev/vibraniumdome", "slug": "genia-dev-vibraniumdome", "url": "https://github.com/genia-dev/vibraniumdome", "owner": "genia-dev", "owner_avatar": "https://avatars.githubusercontent.com/u/140454421?v=4", "stars": 30, "forks": 5, "open_issues": 2, "language": "Python", "license": "GPL-3.0", "pushed_at": "2024-10-28T16:16:04Z", "created_at": "2023-11-29T09:03:25Z", "description": "LLM Security Platform.", "topics": [ "adversarial-attacks", "chatgpt", "large-language-model", "llm", "llm-agent", "llm-eval", "llm-evaluation", "llm-firewall", "llm-framework", "llm-inference", "llm-security", "llm-serving", "llmops", "openai", "prompt-engineering", "prompt-injection", "prompt-injection-tool", "prompts", "security" ], "category": "Red Teaming & Safety", "momentum": 18, "rank": 269 }, { "name": "structured-outputs", "full_name": "thedataquarry/structured-outputs", "slug": "thedataquarry-structured-outputs", "url": "https://github.com/thedataquarry/structured-outputs", "owner": "thedataquarry", "owner_avatar": "https://avatars.githubusercontent.com/u/152195073?v=4", "stars": 29, "forks": 1, "open_issues": 0, "language": "Python", "license": "MIT", "pushed_at": "2025-12-23T13:58:42Z", "created_at": "2025-08-08T01:41:34Z", "description": "Structured output benchmarks comparing DSPy and BAML with different LLMs", "topics": [ "baml", "dspy", "information-extraction", "llm", "llm-eval", "llm-evaluation", "structured-evaluation", "structured-output" ], "category": "Benchmarks", "momentum": 18, "rank": 270 }, { "name": "scenarios", "full_name": "prime-radiant-inc/scenarios", "slug": "prime-radiant-inc-scenarios", "url": "https://github.com/prime-radiant-inc/scenarios", "owner": "prime-radiant-inc", "owner_avatar": "https://avatars.githubusercontent.com/u/253595997?v=4", "stars": 11, "forks": 2, "open_issues": 0, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-06-06T17:59:40Z", "created_at": "2026-01-24T01:36:56Z", "description": "A portable, model-agnostic test suite of declarative YAML scenarios for evaluating AI assistant security against prompt-injection and data-exfiltration attacks, with Python tools and pluggable harnesses.", "topics": [ "ai-security", "evals", "llm", "mcp", "prompt-injection", "python" ], "category": "Benchmarks", "momentum": 18, "rank": 271 }, { "name": "repoagentbench", "full_name": "HumphreySun98/repoagentbench", "slug": "humphreysun98-repoagentbench", "url": "https://github.com/HumphreySun98/repoagentbench", "owner": "HumphreySun98", "owner_avatar": "https://avatars.githubusercontent.com/u/181440142?v=4", "stars": 26, "forks": 0, "open_issues": 2, "language": "Python", "license": "MIT", "pushed_at": "2026-04-30T06:27:42Z", "created_at": "2026-04-29T04:23:55Z", "description": "SWE-bench for your codebase — mine your merged PRs into local, contamination-free coding-agent benchmarks. Adapters: claude-code, aider (Opus 4.7 / GPT-5.5 / Sonnet 4.6 / Gemini 3.1 Pro).", "topics": [ "agent-evals", "ai-agents", "aider", "benchmark", "claude-opus-4-7", "coding-agents", "developer-tools", "gemini-3-1-pro", "gpt-5-5", "llm-eval", "swe-bench" ], "category": "Agent Eval", "momentum": 17, "rank": 272 }, { "name": "ai-product-bench", "full_name": "amplifying-ai/ai-product-bench", "slug": "amplifying-ai-ai-product-bench", "url": "https://github.com/amplifying-ai/ai-product-bench", "owner": "amplifying-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/212717508?v=4", "stars": 24, "forks": 5, "open_issues": 0, "language": "HTML", "license": "", "pushed_at": "2025-05-27T17:57:23Z", "created_at": "2025-05-25T17:42:07Z", "description": "", "topics": [ "dataset", "evals", "llm-eval" ], "category": "Benchmarks", "momentum": 17, "rank": 273 }, { "name": "llm-comparison-backend", "full_name": "Supahands/llm-comparison-backend", "slug": "supahands-llm-comparison-backend", "url": "https://github.com/Supahands/llm-comparison-backend", "owner": "Supahands", "owner_avatar": "https://avatars.githubusercontent.com/u/17401039?v=4", "stars": 23, "forks": 3, "open_issues": 8, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-01-13T20:17:00Z", "created_at": "2024-10-22T04:28:15Z", "description": "This is an opensource project allowing you to compare two LLM's head to head with a given prompt, this section will be regarding the backend of this project, allowing for llm api's to be incorporated and used in the front-end", "topics": [ "ai", "chatgpt", "llm", "llm-api", "llm-comparison", "llm-eval" ], "category": "Eval Frameworks", "momentum": 16, "rank": 274 }, { "name": "realign", "full_name": "honeyhiveai/realign", "slug": "honeyhiveai-realign", "url": "https://github.com/honeyhiveai/realign", "owner": "honeyhiveai", "owner_avatar": "https://avatars.githubusercontent.com/u/137718501?v=4", "stars": 19, "forks": 1, "open_issues": 0, "language": "Python", "license": "MIT", "pushed_at": "2024-12-04T17:41:19Z", "created_at": "2024-07-09T15:44:12Z", "description": "Realign is a testing and simulation framework for AI applications.", "topics": [ "ai", "aiengineering", "alignment", "evaluation", "llm-eval", "llm-evaluation", "llm-evaluation-framework", "llmops", "llms", "prompt-engineering", "rag", "red-teaming", "simulation" ], "category": "Red Teaming & Safety", "momentum": 15, "rank": 275 }, { "name": "prompt-optimizer-skill", "full_name": "geq1fan/prompt-optimizer-skill", "slug": "geq1fan-prompt-optimizer-skill", "url": "https://github.com/geq1fan/prompt-optimizer-skill", "owner": "geq1fan", "owner_avatar": "https://avatars.githubusercontent.com/u/29982379?v=4", "stars": 16, "forks": 1, "open_issues": 7, "language": "JavaScript", "license": "MIT", "pushed_at": "2026-04-13T00:17:19Z", "created_at": "2026-01-10T17:14:04Z", "description": "一个基于 Project Context 进行 Prompt优化 的 Claude Code SKILL，并配置有界面进行迭代优化；Advanced Claude Code skill for systematic prompt engineering. Optimizes prompts using project context, adversarial testing, and iterative evaluation. Built with Wails & Go.", "topics": [ "anthropic", "claude-code", "claude-skill", "developer-tools", "llm-tools", "productivity", "prompt-engineering", "prompt-optimization" ], "category": "Red Teaming & Safety", "momentum": 15, "rank": 276 }, { "name": "llm-coup", "full_name": "khoj-ai/llm-coup", "slug": "khoj-ai-llm-coup", "url": "https://github.com/khoj-ai/llm-coup", "owner": "khoj-ai", "owner_avatar": "https://avatars.githubusercontent.com/u/134046886?v=4", "stars": 14, "forks": 3, "open_issues": 0, "language": "TypeScript", "license": "GPL-3.0", "pushed_at": "2025-08-18T23:53:33Z", "created_at": "2025-08-04T22:58:18Z", "description": "Let LLMs play coup with each other and see who's the best at deception & strategy", "topics": [ "ai", "artificial-intelligence", "coup", "deception", "environment", "games", "llm-benchmarking", "llm-eval", "llm-evaluation", "llms", "simulation" ], "category": "Benchmarks", "momentum": 14, "rank": 277 }, { "name": "eval-tool", "full_name": "Pythagora-io/eval-tool", "slug": "pythagora-io-eval-tool", "url": "https://github.com/Pythagora-io/eval-tool", "owner": "Pythagora-io", "owner_avatar": "https://avatars.githubusercontent.com/u/123263103?v=4", "stars": 12, "forks": 6, "open_issues": 0, "language": "JavaScript", "license": "", "pushed_at": "2024-12-17T14:07:12Z", "created_at": "2024-06-12T07:37:37Z", "description": "Evaluation tool for testing prompts against multiple LLMs", "topics": [], "category": "Eval Frameworks", "momentum": 13, "rank": 278 }, { "name": "GuessArena", "full_name": "IAAR-Shanghai/GuessArena", "slug": "iaar-shanghai-guessarena", "url": "https://github.com/IAAR-Shanghai/GuessArena", "owner": "IAAR-Shanghai", "owner_avatar": "https://avatars.githubusercontent.com/u/150024132?v=4", "stars": 10, "forks": 0, "open_issues": 1, "language": "Python", "license": "Apache-2.0", "pushed_at": "2025-11-15T09:30:20Z", "created_at": "2025-02-15T18:36:03Z", "description": "[ACL 2025] GuessArena: Guess Who I Am? A Self-Adaptive Framework for Evaluating LLMs in Domain-Specific Knowledge and Reasoning", "topics": [ "benchmark", "chatgpt", "deepseek", "domain-specific-eval", "evaluation-framework", "gamearena", "guessarena", "knowledge-evaluation", "large-language-models", "llm-eval", "openai", "qwen", "reasoning-evaluation", "reliable-evaluation" ], "category": "Reasoning", "momentum": 12, "rank": 279 }, { "name": "prediction-powered-ranking", "full_name": "Human-Centric-Machine-Learning/prediction-powered-ranking", "slug": "human-centric-machine-learning-prediction-powered-ranking", "url": "https://github.com/Human-Centric-Machine-Learning/prediction-powered-ranking", "owner": "Human-Centric-Machine-Learning", "owner_avatar": "https://avatars.githubusercontent.com/u/15324966?v=4", "stars": 10, "forks": 1, "open_issues": 0, "language": "Jupyter Notebook", "license": "MIT", "pushed_at": "2024-10-28T10:35:13Z", "created_at": "2024-02-27T13:21:18Z", "description": "Code for \"Prediction-Powered Ranking of Large Language Models\", NeurIPS 2024.", "topics": [ "llm-eval", "llm-evaluation", "llm-evaluation-framework", "prediction-powered-inference", "rank-sets", "ranking-algorithm" ], "category": "Coding Eval", "momentum": 12, "rank": 280 }, { "name": "python-sdk", "full_name": "prompt-foundry/python-sdk", "slug": "prompt-foundry-python-sdk", "url": "https://github.com/prompt-foundry/python-sdk", "owner": "prompt-foundry", "owner_avatar": "https://avatars.githubusercontent.com/u/170367176?v=4", "stars": 8, "forks": 0, "open_issues": 0, "language": "Python", "license": "Apache-2.0", "pushed_at": "2026-05-13T02:40:18Z", "created_at": "2024-06-12T22:00:44Z", "description": "The prompt engineering, prompt management, and prompt evaluation tool for Python", "topics": [ "llm", "llm-eval", "llm-evaluation", "open-ai", "prompt-engineering", "prompt-evaluation", "prompt-management", "python", "python3" ], "category": "Eval Frameworks", "momentum": 12, "rank": 281 } ] }