{ "@context": { "@language": "en", "@vocab": "https://schema.org/", "citeAs": "cr:citeAs", "column": "cr:column", "conformsTo": "dct:conformsTo", "cr": "http://mlcommons.org/croissant/", "rai": "http://mlcommons.org/croissant/RAI/", "data": { "@id": "cr:data", "@type": "@json" }, "dataType": { "@id": "cr:dataType", "@type": "@vocab" }, "dct": "http://purl.org/dc/terms/", "examples": { "@id": "cr:examples", "@type": "@json" }, "extract": "cr:extract", "field": "cr:field", "fileProperty": "cr:fileProperty", "fileObject": "cr:fileObject", "fileSet": "cr:fileSet", "format": "cr:format", "includes": "cr:includes", "isLiveDataset": "cr:isLiveDataset", "jsonPath": "cr:jsonPath", "key": "cr:key", "md5": "cr:md5", "parentField": "cr:parentField", "path": "cr:path", "recordSet": "cr:recordSet", "references": "cr:references", "regex": "cr:regex", "repeated": "cr:repeated", "replace": "cr:replace", "sc": "https://schema.org/", "separator": "cr:separator", "source": "cr:source", "subField": "cr:subField", "transform": "cr:transform" }, "@type": "sc:Dataset", "conformsTo": "http://mlcommons.org/croissant/1.0", "name": "Sycophantic Metacognition", "alternateName": [ "sycophantic-metacognition" ], "description": "A 180-item task battery with Phase 2 pressure items and Phase 3 evidence items; per-model responses for Phase 1 baseline calibration, Phase 2 social-pressure experiments and Phase 3 evidence discrimination; compiled CSVs, calibration metrics and analysis summary; experimental prompts, notebooks, scripts and figures.", "url": "https://doi.org/10.5281/zenodo.24179061", "sameAs": [ "https://huggingface.co/datasets/ai4he/sycophantic-metacognition", "https://github.com/ai4he/kruger", "https://doi.org/10.1145/3816046.3816233" ], "license": "https://creativecommons.org/licenses/by/4.0/", "version": "1.0.0", "datePublished": "2026-11-12", "inLanguage": "en", "isLiveDataset": false, "keywords": [ "metacognition", "calibration", "sycophancy", "dunning-kruger", "llm-evaluation" ], "creator": [ { "@type": "sc:Person", "name": "Manuel Delaflor", "affiliation": { "@type": "sc:Organization", "name": "Metacognition Institute" } }, { "@type": "sc:Person", "name": "Carlos Toxtli", "affiliation": { "@type": "sc:Organization", "name": "Clemson University" }, "identifier": "https://orcid.org/0000-0002-9430-2661" } ], "citeAs": "@inproceedings{Delaflor_2026a,\n series = {CUI ’26},\n title = {Sycophantic Metacognition: Investigating the Dunning-Kruger Effect in Large Language Model Self-Assessment},\n url = {http://dx.doi.org/10.1145/3816046.3816233},\n doi = {10.1145/3816046.3816233},\n booktitle = {Proceedings of the 8th ACM Conference on Conversational User Interfaces},\n publisher = {ACM},\n author = {Delaflor, Manuel and Toxtli, Carlos},\n year = {2026},\n month = July,\n pages = {1--17},\n collection = {CUI ’26}\n}", "distribution": [ { "@type": "cr:FileObject", "@id": "00_research_protocol.ipynb", "name": "00_research_protocol.ipynb", "contentUrl": "00_research_protocol.ipynb", "contentSize": "17812 B", "encodingFormat": "application/x-ipynb+json", "sha256": "d4edaa656caae981352186ce508ee5ec2db59aa2c03ebe92aa5db1857b4bb6fe" }, { "@type": "cr:FileObject", "@id": "01_task_battery.ipynb", "name": "01_task_battery.ipynb", "contentUrl": "01_task_battery.ipynb", "contentSize": "56101 B", "encodingFormat": "application/x-ipynb+json", "sha256": "48bef17b7bf676fb849d4f530438914bfb45fa5c558e72ae102ff33165a75aa9" }, { "@type": "cr:FileObject", "@id": "01_task_battery_executed.ipynb", "name": "01_task_battery_executed.ipynb", "contentUrl": "01_task_battery_executed.ipynb", "contentSize": "63221 B", "encodingFormat": "application/x-ipynb+json", "sha256": "aa5dfd6beb38d8682c4a912faa8a7040d31e10ce85e9a59e7d6bed592621ee85" }, { "@type": "cr:FileObject", "@id": "02_data_collection.ipynb", "name": "02_data_collection.ipynb", "contentUrl": "02_data_collection.ipynb", "contentSize": "24237 B", "encodingFormat": "application/x-ipynb+json", "sha256": "b850b011858147178f414cf02c819603fbcb782fdeaca5e8e3f1a6dd3e6b7910" }, { "@type": "cr:FileObject", "@id": "03_pressure_experiments.ipynb", "name": "03_pressure_experiments.ipynb", "contentUrl": "03_pressure_experiments.ipynb", "contentSize": "22492 B", "encodingFormat": "application/x-ipynb+json", "sha256": "589d6db4daa606a3cd72e7d6289d8f1a5fee31e7a533dcb42d5c0e34b8383e2d" }, { "@type": "cr:FileObject", "@id": "04_analysis.ipynb", "name": "04_analysis.ipynb", "contentUrl": "04_analysis.ipynb", "contentSize": "29864 B", "encodingFormat": "application/x-ipynb+json", "sha256": "438489db07c8f5961ecd7095fcb6acf4fb4045146f1c816fac255f12c987235c" }, { "@type": "cr:FileObject", "@id": "05_nlp_evaluation.ipynb", "name": "05_nlp_evaluation.ipynb", "contentUrl": "05_nlp_evaluation.ipynb", "contentSize": "23635 B", "encodingFormat": "application/x-ipynb+json", "sha256": "8c9b4851794d53d9c3f6905e4d1a577fb3e834518b979f3ee1958a80e93064d6" }, { "@type": "cr:FileObject", "@id": "06_run_experiment.ipynb", "name": "06_run_experiment.ipynb", "contentUrl": "06_run_experiment.ipynb", "contentSize": "13446 B", "encodingFormat": "application/x-ipynb+json", "sha256": "229c7e89d3c5bdaf6787fdc50cb7d1883b59e71056db13210054d0ac903f25be" }, { "@type": "cr:FileObject", "@id": "check_status.sh", "name": "check_status.sh", "contentUrl": "check_status.sh", "contentSize": "3214 B", "encodingFormat": "text/x-shellscript", "sha256": "d477b5d4924d6f373a5bb1cfd868f7ca57a0393cf4b4e32f7240060e82357b23" }, { "@type": "cr:FileObject", "@id": "extract_all_quotes.py", "name": "extract_all_quotes.py", "contentUrl": "extract_all_quotes.py", "contentSize": "17178 B", "encodingFormat": "text/x-python", "sha256": "4d5361fcb9e40ed18a96590ffb48759557584fde5c86ec777f132ff820eca6a2" }, { "@type": "cr:FileObject", "@id": "extract_quotes.py", "name": "extract_quotes.py", "contentUrl": "extract_quotes.py", "contentSize": "3837 B", "encodingFormat": "text/x-python", "sha256": "24e8f038d4b99e5fbe60287f8cd5a8a48dae538da19f8df2a18ee67c6f2ddbe5" }, { "@type": "cr:FileObject", "@id": "find_genuine_wrong.py", "name": "find_genuine_wrong.py", "contentUrl": "find_genuine_wrong.py", "contentSize": "3868 B", "encodingFormat": "text/x-python", "sha256": "df10ff3a8b3c2b0028479d225e3b501178b8f3ed282d1751d5704be666375600" }, { "@type": "cr:FileObject", "@id": "find_p2_p3_all.py", "name": "find_p2_p3_all.py", "contentUrl": "find_p2_p3_all.py", "contentSize": "5432 B", "encodingFormat": "text/x-python", "sha256": "4b54731bb247e2efc109dece1fc8e889325f36140d750fddc89cb051c110a74c" }, { "@type": "cr:FileObject", "@id": "find_pressure_verbose.py", "name": "find_pressure_verbose.py", "contentUrl": "find_pressure_verbose.py", "contentSize": "6025 B", "encodingFormat": "text/x-python", "sha256": "bc66e913b83c0fcba23fc93ce33c45b4da34779c7d89e5325c775cf9642d5a92" }, { "@type": "cr:FileObject", "@id": "find_qwen_pressure.py", "name": "find_qwen_pressure.py", "contentUrl": "find_qwen_pressure.py", "contentSize": "3466 B", "encodingFormat": "text/x-python", "sha256": "54b8e8b04f2ba7198e0dac25743469faa82b85b5d892b4c65aec3a82cee20a5c" }, { "@type": "cr:FileObject", "@id": "run_additional_models.py", "name": "run_additional_models.py", "contentUrl": "run_additional_models.py", "contentSize": "36080 B", "encodingFormat": "text/x-python", "sha256": "f30b51f817fed95602ba83fd297d448e3d8b316ff8c18ec807147f5b2cc94f0d" }, { "@type": "cr:FileObject", "@id": "run_analysis.py", "name": "run_analysis.py", "contentUrl": "run_analysis.py", "contentSize": "55068 B", "encodingFormat": "text/x-python", "sha256": "59b698f2c5e57b6d7fc1ad773ebe9f174f827de550d7cbf39672f0038e107cfe" }, { "@type": "cr:FileObject", "@id": "run_phase1.py", "name": "run_phase1.py", "contentUrl": "run_phase1.py", "contentSize": "16109 B", "encodingFormat": "text/x-python", "sha256": "3e70a7f32c51a91601c2f750586fce19cf4d7e70eaec980c5b4b4025f452644f" }, { "@type": "cr:FileObject", "@id": "run_phase2.py", "name": "run_phase2.py", "contentUrl": "run_phase2.py", "contentSize": "19435 B", "encodingFormat": "text/x-python", "sha256": "fed5381f52fa0e43ffbb2d495af8e68b3a33876acc4db8376efed63421863e9d" }, { "@type": "cr:FileObject", "@id": "run_phase3.py", "name": "run_phase3.py", "contentUrl": "run_phase3.py", "contentSize": "19721 B", "encodingFormat": "text/x-python", "sha256": "c442cf2a23572e5374c7953a9b954e880bedbc760931056740d1803919254f98" }, { "@type": "cr:FileObject", "@id": "data/phase2_items.json", "name": "data/phase2_items.json", "contentUrl": "data/phase2_items.json", "contentSize": "16104 B", "encodingFormat": "application/json", "sha256": "8599dc1c817d2001657e9d80961cb3ffac94cf7a0189f5d6236b7c0956b4c52e" }, { "@type": "cr:FileObject", "@id": "data/phase3_items.json", "name": "data/phase3_items.json", "contentUrl": "data/phase3_items.json", "contentSize": "19537 B", "encodingFormat": "application/json", "sha256": "d0cd8d7676eb78df263eac192bea710dfb8f07b1e3fe3a28c06126b6c6298698" }, { "@type": "cr:FileObject", "@id": "data/task_battery.json", "name": "data/task_battery.json", "contentUrl": "data/task_battery.json", "contentSize": "57365 B", "encodingFormat": "application/json", "sha256": "434052d98283a90e5763369f58f14daea6c19c29d3a10fc79a5c2bfe0594c026" }, { "@type": "cr:FileObject", "@id": "figures/accuracy_confidence_by_difficulty.png", "name": "figures/accuracy_confidence_by_difficulty.png", "contentUrl": "figures/accuracy_confidence_by_difficulty.png", "contentSize": "207674 B", "encodingFormat": "image/png", "sha256": "581a4ea3561a83a41fbb4de691fe1928d43c68b266c070f109122e456be7841e" }, { "@type": "cr:FileObject", "@id": "figures/calibration_by_domain.png", "name": "figures/calibration_by_domain.png", "contentUrl": "figures/calibration_by_domain.png", "contentSize": "439774 B", "encodingFormat": "image/png", "sha256": "af3d27e3c107248fef00eb09b044c983619e551887492bc35cb7e8ed5c0dc493" }, { "@type": "cr:FileObject", "@id": "figures/combined_summary.png", "name": "figures/combined_summary.png", "contentUrl": "figures/combined_summary.png", "contentSize": "631539 B", "encodingFormat": "image/png", "sha256": "43c443f6a6e4a445e75801e7818c91e71f05f5eff46f1170df14a33212625dd5" }, { "@type": "cr:FileObject", "@id": "figures/dunning_kruger_analysis.png", "name": "figures/dunning_kruger_analysis.png", "contentUrl": "figures/dunning_kruger_analysis.png", "contentSize": "392468 B", "encodingFormat": "image/png", "sha256": "6f6c47b9c98cf126beec4f8f628df73b05f41226ce7917f1c178e739b0aadc6c" }, { "@type": "cr:FileObject", "@id": "figures/evidence_discrimination.png", "name": "figures/evidence_discrimination.png", "contentUrl": "figures/evidence_discrimination.png", "contentSize": "386762 B", "encodingFormat": "image/png", "sha256": "3e894fa6b2317004b13a52fe19c9152b6f4b5dd08c879b2ed7762ee59868fbf5" }, { "@type": "cr:FileObject", "@id": "figures/overconfidence_by_domain.png", "name": "figures/overconfidence_by_domain.png", "contentUrl": "figures/overconfidence_by_domain.png", "contentSize": "223748 B", "encodingFormat": "image/png", "sha256": "ddfff824073440e02bd83c35f55f98817764c6ab97034e5d8e287e776b7b11d3" }, { "@type": "cr:FileObject", "@id": "figures/overconfidence_heatmap.png", "name": "figures/overconfidence_heatmap.png", "contentUrl": "figures/overconfidence_heatmap.png", "contentSize": "239357 B", "encodingFormat": "image/png", "sha256": "def1d1759197364f2a812af45bc8a512aa17f008c3967835954d0fbc3f31d3d4" }, { "@type": "cr:FileObject", "@id": "figures/pressure_effects.png", "name": "figures/pressure_effects.png", "contentUrl": "figures/pressure_effects.png", "contentSize": "393204 B", "encodingFormat": "image/png", "sha256": "4967f16659af134682dcf352264e28aa484cc0b0ebefbc532d06884f93b74e58" }, { "@type": "cr:FileObject", "@id": "figures/reliability_diagrams.png", "name": "figures/reliability_diagrams.png", "contentUrl": "figures/reliability_diagrams.png", "contentSize": "230399 B", "encodingFormat": "image/png", "sha256": "b4fcc09018f336c7e710d0ce810bdcc0a16d08cc1d51e99c341d2991b949bb98" }, { "@type": "cr:FileObject", "@id": "figures/sycophantic_asymmetry.png", "name": "figures/sycophantic_asymmetry.png", "contentUrl": "figures/sycophantic_asymmetry.png", "contentSize": "262453 B", "encodingFormat": "image/png", "sha256": "4cd607abe076b2ce9232be9cf82560f0a43ac27c41117aa7dd98bd6aca0f0ff6" }, { "@type": "cr:FileObject", "@id": "paper/paper.pdf", "name": "paper/paper.pdf", "contentUrl": "paper/paper.pdf", "contentSize": "291038 B", "encodingFormat": "application/pdf", "sha256": "02bd1313bcd547c29b9ff1f874767d503464b5e316966d05d59664fd5c3bafc1" }, { "@type": "cr:FileObject", "@id": "paper/paper.tex", "name": "paper/paper.tex", "contentUrl": "paper/paper.tex", "contentSize": "97476 B", "encodingFormat": "application/x-tex", "sha256": "6962d8545eb699a5ec7e017126b30980a6ca0a53c5e987e8eab17cacef373241" }, { "@type": "cr:FileObject", "@id": "paper/references.bib", "name": "paper/references.bib", "contentUrl": "paper/references.bib", "contentSize": "44076 B", "encodingFormat": "application/x-bibtex", "sha256": "25a4a23dba1612eb2521d0e958df07656a2bdff414a74b9c825f7d4596043f5b" }, { "@type": "cr:FileObject", "@id": "prompts/01_answer_prompt.md", "name": "prompts/01_answer_prompt.md", "contentUrl": "prompts/01_answer_prompt.md", "contentSize": "1251 B", "encodingFormat": "text/markdown", "sha256": "283c0e8176284a4d176b44661e1758b0c592da4e6baba44696532d7629c0b6ac" }, { "@type": "cr:FileObject", "@id": "prompts/02_metacognitive_probes.md", "name": "prompts/02_metacognitive_probes.md", "contentUrl": "prompts/02_metacognitive_probes.md", "contentSize": "2394 B", "encodingFormat": "text/markdown", "sha256": "e3130d9e682631f0240d519b237fa7baabc6a231389e3844cbecebe0c03c45bf" }, { "@type": "cr:FileObject", "@id": "prompts/03_pressure_templates.md", "name": "prompts/03_pressure_templates.md", "contentUrl": "prompts/03_pressure_templates.md", "contentSize": "3649 B", "encodingFormat": "text/markdown", "sha256": "b8dfba1aaa12ce04c432d0593aa4d1a4f549896c2e9435b9d35fa4ee112b1798" }, { "@type": "cr:FileObject", "@id": "prompts/04_evidence_templates.md", "name": "prompts/04_evidence_templates.md", "contentUrl": "prompts/04_evidence_templates.md", "contentSize": "2780 B", "encodingFormat": "text/markdown", "sha256": "c094a0575468a5b0b6df4d0da5b7d1e941ec763ad13ce4a790bfb02cc927e85f" }, { "@type": "cr:FileObject", "@id": "results/analysis_summary.json", "name": "results/analysis_summary.json", "contentUrl": "results/analysis_summary.json", "contentSize": "14836 B", "encodingFormat": "application/json", "sha256": "7aae8a4d2b1924dad682096aaff5ef1d347fffe179c5d7a08f0cbb18ff3d6461" }, { "@type": "cr:FileObject", "@id": "results/calibration_metrics.csv", "name": "results/calibration_metrics.csv", "contentUrl": "results/calibration_metrics.csv", "contentSize": "775 B", "encodingFormat": "text/csv", "sha256": "ecd2f6cda85189562d717d2cb91228e2f563c08e1b9999afed8eeeae79242823" }, { "@type": "cr:FileObject", "@id": "results/phase1_compiled.csv", "name": "results/phase1_compiled.csv", "contentUrl": "results/phase1_compiled.csv", "contentSize": "28151692 B", "encodingFormat": "text/csv", "sha256": "ba775816e3df57e530ac5f63b55d9a1ad358cdaed7e981b0ebecbf67f76009d9" }, { "@type": "cr:FileObject", "@id": "results/phase1_gemini-2.5-flash-lite.json", "name": "results/phase1_gemini-2.5-flash-lite.json", "contentUrl": "results/phase1_gemini-2.5-flash-lite.json", "contentSize": "6175647 B", "encodingFormat": "application/json", "sha256": "0f4dc06a58a81f26dc68ccceb8e67df1a2af1ae2981f621767baa4f0c13c51db" }, { "@type": "cr:FileObject", "@id": "results/phase1_gemini-3-flash-preview.json", "name": "results/phase1_gemini-3-flash-preview.json", "contentUrl": "results/phase1_gemini-3-flash-preview.json", "contentSize": "5653077 B", "encodingFormat": "application/json", "sha256": "5c38d111a618fe716c86ddf3b1bfca506f7f98c5cf968a7aa733a04ab2cf8a2a" }, { "@type": "cr:FileObject", "@id": "results/phase1_gptoss-20b.json", "name": "results/phase1_gptoss-20b.json", "contentUrl": "results/phase1_gptoss-20b.json", "contentSize": "6707411 B", "encodingFormat": "application/json", "sha256": "3ea3b7b4e46c07920ae10e5d8847eb22eb60e532a5cc87b1ac9abf374264c815" }, { "@type": "cr:FileObject", "@id": "results/phase1_ministral-3b.json", "name": "results/phase1_ministral-3b.json", "contentUrl": "results/phase1_ministral-3b.json", "contentSize": "11493183 B", "encodingFormat": "application/json", "sha256": "a56c6e7d4b2f6cdc8de1699dfada04ceeee57465fc60cd1aa638e9ee2c4702b8" }, { "@type": "cr:FileObject", "@id": "results/phase1_qwen3-30b.json", "name": "results/phase1_qwen3-30b.json", "contentUrl": "results/phase1_qwen3-30b.json", "contentSize": "6543503 B", "encodingFormat": "application/json", "sha256": "f47ab00838acda2b47d072513e8c3f1445b4beee42fb618ea3596d7fc73cee75" }, { "@type": "cr:FileObject", "@id": "results/phase2_compiled.csv", "name": "results/phase2_compiled.csv", "contentUrl": "results/phase2_compiled.csv", "contentSize": "4038802 B", "encodingFormat": "text/csv", "sha256": "0f310395e4ba15bc90db7a292d2cc8890205bab38aa2bb0cfab77c1654fac978" }, { "@type": "cr:FileObject", "@id": "results/phase2_gemini-2.5-flash-lite.json", "name": "results/phase2_gemini-2.5-flash-lite.json", "contentUrl": "results/phase2_gemini-2.5-flash-lite.json", "contentSize": "2058019 B", "encodingFormat": "application/json", "sha256": "200e245f574b20f60796d326684c67e6a1a85831a8aa158da16678ba805f68a7" }, { "@type": "cr:FileObject", "@id": "results/phase2_gptoss-20b.json", "name": "results/phase2_gptoss-20b.json", "contentUrl": "results/phase2_gptoss-20b.json", "contentSize": "1975575 B", "encodingFormat": "application/json", "sha256": "24373688d86ddca44a751aedf567e5d2ee8e1ff2979a798ba34e5eaabebd0498" }, { "@type": "cr:FileObject", "@id": "results/phase2_ministral-3b.json", "name": "results/phase2_ministral-3b.json", "contentUrl": "results/phase2_ministral-3b.json", "contentSize": "2331914 B", "encodingFormat": "application/json", "sha256": "7960f9cbc37a04d4fd2d02a45fdc47eabb89bba72b0e03c77552a4e6ed6dc8b4" }, { "@type": "cr:FileObject", "@id": "results/phase2_qwen3-30b.json", "name": "results/phase2_qwen3-30b.json", "contentUrl": "results/phase2_qwen3-30b.json", "contentSize": "2004623 B", "encodingFormat": "application/json", "sha256": "3e8afb8832c0dfe86fa1d95af2be608733341130e4b0927df40f0a0fddb98555" }, { "@type": "cr:FileObject", "@id": "results/phase3_compiled.csv", "name": "results/phase3_compiled.csv", "contentUrl": "results/phase3_compiled.csv", "contentSize": "518344 B", "encodingFormat": "text/csv", "sha256": "9a29c539718d4b3b85f85fb178b59257407e35f97bf0e271b344579d52f70601" }, { "@type": "cr:FileObject", "@id": "results/phase3_gemini-2.5-flash-lite.json", "name": "results/phase3_gemini-2.5-flash-lite.json", "contentUrl": "results/phase3_gemini-2.5-flash-lite.json", "contentSize": "305770 B", "encodingFormat": "application/json", "sha256": "24994d23bddf6ca8a3fe56d58b68fbcc9742555e54224a1133c1d5c1472b9e7a" }, { "@type": "cr:FileObject", "@id": "results/phase3_gptoss-20b.json", "name": "results/phase3_gptoss-20b.json", "contentUrl": "results/phase3_gptoss-20b.json", "contentSize": "297279 B", "encodingFormat": "application/json", "sha256": "0416fab33fe254f96b62e22733b83e3f7331a387994ddff34082bc4b8a8b8a0d" }, { "@type": "cr:FileObject", "@id": "results/phase3_ministral-3b.json", "name": "results/phase3_ministral-3b.json", "contentUrl": "results/phase3_ministral-3b.json", "contentSize": "365485 B", "encodingFormat": "application/json", "sha256": "ded3edffc0faba61d99a444ce7d044692c2439a46b66d33d52611902c0236fb5" }, { "@type": "cr:FileObject", "@id": "results/phase3_qwen3-30b.json", "name": "results/phase3_qwen3-30b.json", "contentUrl": "results/phase3_qwen3-30b.json", "contentSize": "288436 B", "encodingFormat": "application/json", "sha256": "657d1b961b9f31336ab34e0eb120e4eee0fc6d4d5cb6a2d2aa64887a8b943c4f" } ], "recordSet": [ { "@type": "cr:RecordSet", "@id": "results_phase1_compiled", "name": "results_phase1_compiled", "description": "Records from results/phase1_compiled.csv (3,600 rows).", "field": [ { "@type": "cr:Field", "@id": "results_phase1_compiled/model", "name": "model", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "model" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/item_id", "name": "item_id", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "item_id" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/trial", "name": "trial", "dataType": "sc:Integer", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "trial" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/domain", "name": "domain", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "domain" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/difficulty", "name": "difficulty", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "difficulty" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/question", "name": "question", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "question" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/ground_truth", "name": "ground_truth", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "ground_truth" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/answer_type", "name": "answer_type", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "answer_type" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/timestamp", "name": "timestamp", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "timestamp" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/phase", "name": "phase", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "phase" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/answer", "name": "answer", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "answer" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/initial_confidence", "name": "initial_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "initial_confidence" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/raw_answer_response", "name": "raw_answer_response", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "raw_answer_response" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_confidence", "name": "meta_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_confidence" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_confidence_raw", "name": "meta_confidence_raw", "dataType": "sc:Integer", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_confidence_raw" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_justification", "name": "meta_justification", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_justification" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_justification_raw", "name": "meta_justification_raw", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_justification_raw" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_error_awareness", "name": "meta_error_awareness", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_error_awareness" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_error_awareness_raw", "name": "meta_error_awareness_raw", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_error_awareness_raw" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_optimization", "name": "meta_optimization", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_optimization" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_optimization_raw", "name": "meta_optimization_raw", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_optimization_raw" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_difficulty_estimate", "name": "meta_difficulty_estimate", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_difficulty_estimate" } } }, { "@type": "cr:Field", "@id": "results_phase1_compiled/meta_difficulty_estimate_raw", "name": "meta_difficulty_estimate_raw", "dataType": "sc:Integer", "source": { "fileObject": { "@id": "results/phase1_compiled.csv" }, "extract": { "column": "meta_difficulty_estimate_raw" } } } ] }, { "@type": "cr:RecordSet", "@id": "results_phase2_compiled", "name": "results_phase2_compiled", "description": "Records from results/phase2_compiled.csv (9,600 rows).", "field": [ { "@type": "cr:Field", "@id": "results_phase2_compiled/model", "name": "model", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "model" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/item_id", "name": "item_id", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "item_id" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/trial", "name": "trial", "dataType": "sc:Integer", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "trial" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/domain", "name": "domain", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "domain" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/difficulty", "name": "difficulty", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "difficulty" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/question", "name": "question", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "question" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/ground_truth", "name": "ground_truth", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "ground_truth" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/answer_type", "name": "answer_type", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "answer_type" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/condition", "name": "condition", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "condition" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/direction", "name": "direction", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "direction" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/phase", "name": "phase", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "phase" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/timestamp", "name": "timestamp", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "timestamp" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/initial_answer", "name": "initial_answer", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "initial_answer" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/initial_confidence", "name": "initial_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "initial_confidence" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/raw_initial_response", "name": "raw_initial_response", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "raw_initial_response" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/revised_answer", "name": "revised_answer", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "revised_answer" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/revised_confidence", "name": "revised_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "revised_confidence" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/raw_pressure_response", "name": "raw_pressure_response", "dataType": "sc:Integer", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "raw_pressure_response" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/answer_changed", "name": "answer_changed", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "answer_changed" } } }, { "@type": "cr:Field", "@id": "results_phase2_compiled/confidence_shift", "name": "confidence_shift", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase2_compiled.csv" }, "extract": { "column": "confidence_shift" } } } ] }, { "@type": "cr:RecordSet", "@id": "results_phase3_compiled", "name": "results_phase3_compiled", "description": "Records from results/phase3_compiled.csv (1,600 rows).", "field": [ { "@type": "cr:Field", "@id": "results_phase3_compiled/model", "name": "model", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "model" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/item_id", "name": "item_id", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "item_id" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/trial", "name": "trial", "dataType": "sc:Integer", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "trial" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/domain", "name": "domain", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "domain" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/difficulty", "name": "difficulty", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "difficulty" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/question", "name": "question", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "question" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/ground_truth", "name": "ground_truth", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "ground_truth" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/answer_type", "name": "answer_type", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "answer_type" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/alternative_answer", "name": "alternative_answer", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "alternative_answer" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/condition", "name": "condition", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "condition" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/phase", "name": "phase", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "phase" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/timestamp", "name": "timestamp", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "timestamp" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/initial_answer", "name": "initial_answer", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "initial_answer" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/initial_confidence", "name": "initial_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "initial_confidence" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/raw_initial_response", "name": "raw_initial_response", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "raw_initial_response" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/revised_answer", "name": "revised_answer", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "revised_answer" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/revised_confidence", "name": "revised_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "revised_confidence" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/raw_challenge_response", "name": "raw_challenge_response", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "raw_challenge_response" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/answer_changed", "name": "answer_changed", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "answer_changed" } } }, { "@type": "cr:Field", "@id": "results_phase3_compiled/confidence_shift", "name": "confidence_shift", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/phase3_compiled.csv" }, "extract": { "column": "confidence_shift" } } } ] }, { "@type": "cr:RecordSet", "@id": "results_calibration_metrics", "name": "results_calibration_metrics", "description": "Records from results/calibration_metrics.csv (4 rows).", "field": [ { "@type": "cr:Field", "@id": "results_calibration_metrics/model", "name": "model", "dataType": "sc:Text", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "model" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/ece", "name": "ece", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "ece" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/mce", "name": "mce", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "mce" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/brier", "name": "brier", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "brier" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/mean_confidence", "name": "mean_confidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "mean_confidence" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/mean_accuracy", "name": "mean_accuracy", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "mean_accuracy" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/overconfidence", "name": "overconfidence", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "overconfidence" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/spearman_rho", "name": "spearman_rho", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "spearman_rho" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/spearman_p", "name": "spearman_p", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "spearman_p" } } }, { "@type": "cr:Field", "@id": "results_calibration_metrics/n_observations", "name": "n_observations", "dataType": "sc:Float", "source": { "fileObject": { "@id": "results/calibration_metrics.csv" }, "extract": { "column": "n_observations" } } } ] } ] }