@article{04379ca515864793862de1cd2a69e0c0,
title = "Establishing a training plan and estimating inter-rater reliability across the multi-site Texas childhood trauma research network",
abstract = "Objective: Minimal guidance is available in the literature to develop protocols for training non-clinician raters to administer semi-structured psychiatric interviews in large, multi-site studies. Previous work has not produced standardized methods for maintaining rater quality control or estimating interrater reliability (IRR) in such studies. Our objective is to describe the multi-site Texas Childhood Trauma Research Network (TX-CTRN) rater training protocol and activities used to maintain rater calibration and evaluate protocol effectiveness. Methods: Rater training utilized synchronous and asynchronous didactic learning modules, and certification involved critique of videotaped mock scale administration. Certified raters attended monthly review meetings and completed ongoing scoring exercises for quality assurance purposes. Training protocol effectiveness was evaluated using individual measure and pooled estimated IRRs for three key study measures (TESI-C, CAPS-CA-5, MINI-KID [Major Depressive Episodes - MDE & Posttraumatic Stress Disorder – PTSD modules]). A random selection of video-recorded administrations of these measures was evaluated by three certified raters to estimate agreement statistics, with jackknife (on the videos) used for confidence interval estimation. Kappa, weighted kappa and intraclass correlations were calculated for study measure ratings. Results: IRR agreement across all measures was strong (TESI-C median kappa 0.79, lower 95% CB 0.66; CAPS-CA-5 median weighted kappa 0.71 (0.62), MINI-MDE median kappa 0.71 (0.62), MINI-PTSD median kappa 0.91 (0.9). The combined estimated ICC was ≥0.86 (lower CBs ≥0.69). Conclusions: The protocol developed by TX-CTRN may serve as a model for other multi-site studies that require comprehensive non-clinician rater training, quality assurance guidelines, and a system for assessing and estimating IRR.",
keywords = "Inter-rater reliability, Measurement, Psychiatry research, Reliability, Training, Trauma",
author = "Shahidullah, {Jeffrey D.} and James Custer and Oscar Widales-Benitez and Nazan Aksan and Carly Hatchell and Newport, {D. Jeffrey} and Wagner, {Karen Dineen} and Storch, {Eric A.} and Cynthia Claassen and Amy Garrett and Ugalde, {Irma T.} and Wade Weber and Nemeroff, {Charles B.} and Rathouz, {Paul J.}",
note = "Funding Information: Dr. Eric Storch receives grant support from NIH, the Ream Foundation, Greater Houston Community Foundation, International OCD Foundation, and Texas Higher Education Coordinating Board. He receives book royalties from Elsevier, Springer, American Psychological Association, Jessica Kingsley, Oxford, and Lawrence Erlbaum. He holds stock in NView, where he serves on the clinical advisory board. He was a consultant for Levo Therapeutics, and is currently a consultant for Biohaven Pharmaceuticals and Brainsway. He co-founded and receives payment from Rethinking Behavioral Health, which is a consulting firm that provides support for implementing evidence-based psychological treatment strategies.Dr. Nemeroff has received research support from National Institutes of Health (NIH). He has served as a consultant to AbbVie, ANeuroTech (division of Anima BV), Signant Health, Magstim, Inc., Intra-Cellular Therapies, Inc., EMA Wellness, Sage, Silo Pharma, Engrail Therapeutics, Pasithea Therapeutic Corp., EcoR1, GoodCap Pharmaceuticals, Inc., Senseye, Clexio, Ninnion Therapeutics, AncoraBio, SynapseBio, BioXcel Therapeutics. He is a stockholder with Seattle Genetics, Antares, Inc., Corcept Therapeutics Pharmaceuticals Company, EMA Wellness, Naki Health, Relmada Therapeutics. He has served on advisory boards for ANeuroTech (division of Anima BV), Brain and Behavior Research Foundation (BBRF), Anxiety and Depression Association of America (ADAA), Skyland Trail, Signant Health, Laureate Institute for Brain Research (LIBR), Inc., Heading Health, Pasithea Therapeutic Corp., Sage. He has served on the Board of Directors for Gratitude America, ADAA, Lucy Scientific Discovery, Inc. He holds the following patents: Method and devices for transdermal delivery of lithium (US 6375,990B1); Method of assessing antidepressant drug therapy via transport inhibition of monoamine neurotransmitters by ex vivo assay (US 7148,027B2).Dr. Claassen has received support from the National Institutes of Health (NIH), the Borderline Research Foundation, Timberlawn Foundation and the Jordan Elizabeth Harris Foundation. She currently serves as a consultant to LivaNova PLC. Publisher Copyright: {\textcopyright} 2023 Elsevier B.V.",
year = "2023",
month = may,
doi = "10.1016/j.psychres.2023.115168",
language = "English (US)",
volume = "323",
journal = "Psychiatry Research",
issn = "0165-1781",
publisher = "Elsevier Ireland Ltd",
}