CITATION.bib

@inproceedings{10.1145/3459637.3482099,
author = {Pathiyan Cherumanal, Sachin and Spina, Damiano and Scholer, Falk and Croft, W. Bruce},
title = {Evaluating Fairness in Argument Retrieval},
year = {2021},
isbn = {9781450384469},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3459637.3482099},
doi = {10.1145/3459637.3482099},
abstract = {Existing commercial search engines often struggle to represent different perspectives
of a search query. Argument retrieval systems address this limitation of search engines
and provide both positive (PRO) and negative (CON) perspectives about a user's information
need on a controversial topic (e.g., climate change). The effectiveness of such argument
retrieval systems is typically evaluated based on topical relevance and argument quality,
without taking into account the often differing number of documents shown for the
argument stances (PRO or CON). Therefore, systems may retrieve relevant passages,
but with a biased exposure of arguments. In this work, we analyze a range of non-stochastic
fairness-aware ranking and diversity metrics to evaluate the extent to which argument
stances are fairly exposed in argument retrieval systems.Using the official runs of
the argument retrieval task Ttouch\'{e} at CLEF 2020, as well as synthetic data to control
the amount and order of argument stances in the rankings, we show that systems with
the best effectiveness in terms of topical relevance are not necessarily the most
fair or the most diverse in terms of argument stance. The relationships we found between
(un)fairness and diversity metrics shed light on how to evaluate group fairness --
in addition to topical relevance -- in argument retrieval settings.},
booktitle = {Proceedings of the 30th ACM International Conference on Information & Knowledge Management},
pages = {3363–3367},
numpages = {5},
keywords = {evaluation, fairness, argument retrieval},
location = {Virtual Event, Queensland, Australia},
series = {CIKM '21}
}