@inproceedings{FormalDistillationHardNegative2022,
 abstract = {Neural retrievers based on dense representations combined with Approximate Nearest Neighbors search have recently received a lot of attention, owing their success to distillation and/or better sampling of examples for training -- while still relying on the same backbone architecture. In the meantime, sparse representation learning fueled by traditional inverted indexing techniques has seen a growing interest, inheriting from desirable IR priors such as explicit lexical matching. While some architectural variants have been proposed, a lesser effort has been put in the training of such models. In this work, we build on SPLADE -- a sparse expansion-based retriever -- and show to which extent it is able to benefit from the same training improvements as dense models, by studying the effect of distillation, hard-negative mining as well as the Pre-trained Language Model initialization. We furthermore study the link between effectiveness and efficiency, on in-domain and zero-shot settings, leading to state-of-the-art results in both scenarios for sufficiently expressive models.},
 address = {New York, NY, USA},
 author = {Formal, Thibault and Lassance, Carlos and Piwowarski, Benjamin and Clinchant, Stéphane},
 booktitle = {Proceedings of the 45th {International} {ACM} {SIGIR} {Conference} on {Research} and {Development} in {Information} {Retrieval}},
 copyright = {All rights reserved},
 doi = {10.1145/3477495.3531857},
 isbn = {978-1-4503-8732-3},
 month = {July},
 pages = {2353--2359},
 publisher = {Association for Computing Machinery},
 series = {{SIGIR} '22},
 shorttitle = {From {Distillation} to {Hard} {Negative} {Sampling}},
 title = {From {Distillation} to {Hard} {Negative} {Sampling}: {Making} {Sparse} {Neural} {IR} {Models} {More} {Effective}},
 url = {http://doi.org/10.1145/3477495.3531857},
 urldate = {2022-07-21},
 year = {2022}
}
