@article{li_zhang_luo_cai_fang_yuan_2022, 
  title={Do Pre-trained Language Models Indeed Understand Software Engineering   Tasks?}, 
  abstractNote={Artificial intelligence (AI) for software engineering (SE) tasks has recently
achieved promising performance. In this paper, we investigate to what extent
the pre-trained language model truly understands those SE tasks such as code
search, code summarization, etc. We conduct a comprehensive empirical study on
a board set of AI for SE (AI4SE) tasks by feeding them with variant inputs: 1)
with various masking rates and 2) with sufficient input subset method. Then,
the trained models are evaluated on different SE tasks, including code search,
code summarization, and duplicate bug report detection. Our experimental
results show that pre-trained language models are insensitive to the given
input, thus they achieve similar performance in these three SE tasks. We refer
to this phenomenon as overinterpretation, where a model confidently makes a
decision without salient features, or where a model finds some irrelevant
relationships between the final decision and the dataset. Our study
investigates two approaches to mitigate the overinterpretation phenomenon:
whole word mask strategy and ensembling. To the best of our knowledge, we are
the first to reveal this overinterpretation phenomenon to the AI4SE community,
which is an important reminder for researchers to design the input for the
models and calls for necessary future work in understanding and implementing
AI4SE tasks.}, 
  author={Li and Zhang and Luo and Cai and Fang and Yuan}, 
  year={2022}, 
  month={Nov}
  }