@article{lin-etal-2023-aggretriever,
title = "Aggretriever: A Simple Approach to Aggregate Textual Representations for Robust Dense Passage Retrieval",
author = "Lin, Sheng-Chieh and
Li, Minghan and
Lin, Jimmy",
journal = "Transactions of the Association for Computational Linguistics",
volume = "11",
year = "2023",
address = "Cambridge, MA",
publisher = "MIT Press",
url = "https://aclanthology.org/2023.tacl-1.26/",
doi = "10.1162/tacl_a_00556",
pages = "436--452",
abstract = "Pre-trained language models have been successful in many knowledge-intensive NLP tasks. However, recent work has shown that models such as BERT are not {\textquotedblleft}structurally ready{\textquotedblright} to aggregate textual information into a [CLS] vector for dense passage retrieval (DPR). This {\textquotedblleft}lack of readiness{\textquotedblright} results from the gap between language model pre-training and DPR fine-tuning. Previous solutions call for computationally expensive techniques such as hard negative mining, cross-encoder distillation, and further pre-training to learn a robust DPR model. In this work, we instead propose to fully exploit knowledge in a pre-trained language model for DPR by aggregating the contextualized token embeddings into a dense vector, which we call agg★. By concatenating vectors from the [CLS] token and agg★, our Aggretriever model substantially improves the effectiveness of dense retrieval models on both in-domain and zero-shot evaluations without introducing substantial training overhead. Code is available at \url{https://github.com/castorini/dhr}."
}<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="lin-etal-2023-aggretriever">
<titleInfo>
<title>Aggretriever: A Simple Approach to Aggregate Textual Representations for Robust Dense Passage Retrieval</title>
</titleInfo>
<name type="personal">
<namePart type="given">Sheng-Chieh</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Minghan</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jimmy</namePart>
<namePart type="family">Lin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Transactions of the Association for Computational Linguistics</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>MIT Press</publisher>
<place>
<placeTerm type="text">Cambridge, MA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Pre-trained language models have been successful in many knowledge-intensive NLP tasks. However, recent work has shown that models such as BERT are not “structurally ready” to aggregate textual information into a [CLS] vector for dense passage retrieval (DPR). This “lack of readiness” results from the gap between language model pre-training and DPR fine-tuning. Previous solutions call for computationally expensive techniques such as hard negative mining, cross-encoder distillation, and further pre-training to learn a robust DPR model. In this work, we instead propose to fully exploit knowledge in a pre-trained language model for DPR by aggregating the contextualized token embeddings into a dense vector, which we call agg★. By concatenating vectors from the [CLS] token and agg★, our Aggretriever model substantially improves the effectiveness of dense retrieval models on both in-domain and zero-shot evaluations without introducing substantial training overhead. Code is available at https://github.com/castorini/dhr.</abstract>
<identifier type="citekey">lin-etal-2023-aggretriever</identifier>
<identifier type="doi">10.1162/tacl_a_00556</identifier>
<location>
<url>https://aclanthology.org/2023.tacl-1.26/</url>
</location>
<part>
<date>2023</date>
<detail type="volume"><number>11</number></detail>
<extent unit="page">
<start>436</start>
<end>452</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T Aggretriever: A Simple Approach to Aggregate Textual Representations for Robust Dense Passage Retrieval
%A Lin, Sheng-Chieh
%A Li, Minghan
%A Lin, Jimmy
%J Transactions of the Association for Computational Linguistics
%D 2023
%V 11
%I MIT Press
%C Cambridge, MA
%F lin-etal-2023-aggretriever
%X Pre-trained language models have been successful in many knowledge-intensive NLP tasks. However, recent work has shown that models such as BERT are not “structurally ready” to aggregate textual information into a [CLS] vector for dense passage retrieval (DPR). This “lack of readiness” results from the gap between language model pre-training and DPR fine-tuning. Previous solutions call for computationally expensive techniques such as hard negative mining, cross-encoder distillation, and further pre-training to learn a robust DPR model. In this work, we instead propose to fully exploit knowledge in a pre-trained language model for DPR by aggregating the contextualized token embeddings into a dense vector, which we call agg★. By concatenating vectors from the [CLS] token and agg★, our Aggretriever model substantially improves the effectiveness of dense retrieval models on both in-domain and zero-shot evaluations without introducing substantial training overhead. Code is available at https://github.com/castorini/dhr.
%R 10.1162/tacl_a_00556
%U https://aclanthology.org/2023.tacl-1.26/
%U https://doi.org/10.1162/tacl_a_00556
%P 436-452
Markdown (Informal)
[Aggretriever: A Simple Approach to Aggregate Textual Representations for Robust Dense Passage Retrieval](https://aclanthology.org/2023.tacl-1.26/) (Lin et al., TACL 2023)
ACL