@inproceedings{mu-etal-2023-time,
title = "It{'}s about Time: Rethinking Evaluation on Rumor Detection Benchmarks using Chronological Splits",
author = "Mu, Yida and
Bontcheva, Kalina and
Aletras, Nikolaos",
editor = "Vlachos, Andreas and
Augenstein, Isabelle",
booktitle = "Findings of the Association for Computational Linguistics: EACL 2023",
month = may,
year = "2023",
address = "Dubrovnik, Croatia",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.findings-eacl.55",
doi = "10.18653/v1/2023.findings-eacl.55",
pages = "736--743",
abstract = "New events emerge over time influencing the topics of rumors in social media. Current rumor detection benchmarks use random splits as training, development and test sets which typically results in topical overlaps. Consequently, models trained on random splits may not perform well on rumor classification on previously unseen topics due to the temporal concept drift. In this paper, we provide a re-evaluation of classification models on four popular rumor detection benchmarks considering chronological instead of random splits. Our experimental results show that the use of random splits can significantly overestimate predictive performance across all datasets and models. Therefore, we suggest that rumor detection models should always be evaluated using chronological splits for minimizing topical overlaps.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="mu-etal-2023-time">
<titleInfo>
<title>It’s about Time: Rethinking Evaluation on Rumor Detection Benchmarks using Chronological Splits</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yida</namePart>
<namePart type="family">Mu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Kalina</namePart>
<namePart type="family">Bontcheva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nikolaos</namePart>
<namePart type="family">Aletras</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-05</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: EACL 2023</title>
</titleInfo>
<name type="personal">
<namePart type="given">Andreas</namePart>
<namePart type="family">Vlachos</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Isabelle</namePart>
<namePart type="family">Augenstein</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Dubrovnik, Croatia</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>New events emerge over time influencing the topics of rumors in social media. Current rumor detection benchmarks use random splits as training, development and test sets which typically results in topical overlaps. Consequently, models trained on random splits may not perform well on rumor classification on previously unseen topics due to the temporal concept drift. In this paper, we provide a re-evaluation of classification models on four popular rumor detection benchmarks considering chronological instead of random splits. Our experimental results show that the use of random splits can significantly overestimate predictive performance across all datasets and models. Therefore, we suggest that rumor detection models should always be evaluated using chronological splits for minimizing topical overlaps.</abstract>
<identifier type="citekey">mu-etal-2023-time</identifier>
<identifier type="doi">10.18653/v1/2023.findings-eacl.55</identifier>
<location>
<url>https://aclanthology.org/2023.findings-eacl.55</url>
</location>
<part>
<date>2023-05</date>
<extent unit="page">
<start>736</start>
<end>743</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T It’s about Time: Rethinking Evaluation on Rumor Detection Benchmarks using Chronological Splits
%A Mu, Yida
%A Bontcheva, Kalina
%A Aletras, Nikolaos
%Y Vlachos, Andreas
%Y Augenstein, Isabelle
%S Findings of the Association for Computational Linguistics: EACL 2023
%D 2023
%8 May
%I Association for Computational Linguistics
%C Dubrovnik, Croatia
%F mu-etal-2023-time
%X New events emerge over time influencing the topics of rumors in social media. Current rumor detection benchmarks use random splits as training, development and test sets which typically results in topical overlaps. Consequently, models trained on random splits may not perform well on rumor classification on previously unseen topics due to the temporal concept drift. In this paper, we provide a re-evaluation of classification models on four popular rumor detection benchmarks considering chronological instead of random splits. Our experimental results show that the use of random splits can significantly overestimate predictive performance across all datasets and models. Therefore, we suggest that rumor detection models should always be evaluated using chronological splits for minimizing topical overlaps.
%R 10.18653/v1/2023.findings-eacl.55
%U https://aclanthology.org/2023.findings-eacl.55
%U https://doi.org/10.18653/v1/2023.findings-eacl.55
%P 736-743
Markdown (Informal)
[It’s about Time: Rethinking Evaluation on Rumor Detection Benchmarks using Chronological Splits](https://aclanthology.org/2023.findings-eacl.55) (Mu et al., Findings 2023)
ACL