Spaces:
Runtime error
Runtime error
File size: 3,762 Bytes
5fc6e5d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
import pytest
# These tests check that "noise" (like capitalization or punctuation) does not change the prediction
@pytest.mark.parametrize(
"comment",
[
":param user_id: The ID of the user.", # Base
":PARAM USER_ID: THE ID OF THE USER.", # Uppercase
" :param user_id: The ID of the user . ", # Whitespace
":param user_id: The ID of the user!!!", # Punctuation
],
)
def test_python_invariance_parameters(python_model, comment, get_predicted_labels):
"""Tests that noise doesn't break ':param' detection."""
expected = {"Parameters"}
preds = get_predicted_labels(python_model, comment, "python")
assert preds == expected
def test_java_invariance_deprecation(java_model, get_predicted_labels):
"""Tests that noise doesn't break '@deprecated' detection"""
base_comment = "/** @deprecated Use newUserMethod() */"
pert_comment = "/** @DEPRECATED... Use newUserMethod()!!! */"
preds_base = get_predicted_labels(java_model, base_comment, "java")
preds_pert = get_predicted_labels(java_model, pert_comment, "java")
assert {"deprecation"} <= preds_base
assert preds_base == preds_pert
def test_python_invariance_summary(python_model, get_predicted_labels):
"""Tests that noise doesn't break a simple 'Summary' detection"""
base_comment = "a service specific account of type bar."
expected = {"Summary"}
# Perturbations
variants = [
base_comment,
"A SERVICE SPECIFIC ACCOUNT OF TYPE BAR.",
" a service specific account of type bar. ",
"a service specific account of type bar!!!",
]
for comment in variants:
preds = get_predicted_labels(python_model, comment, "python")
assert preds == expected
def test_pharo_invariance_intent(pharo_model, get_predicted_labels):
"""Tests that noise doesn't break Pharo's 'Intent' detection"""
base_comment = '"i am a simple arrow like arrowhead."'
expected = {"Intent"}
# Perturbations
variants = [
base_comment,
'"I AM A SIMPLE ARROW LIKE ARROWHEAD."',
' "i am a simple arrow like arrowhead." ',
'"i am a simple arrow like arrowhead !!"', #
]
for comment in variants:
preds = get_predicted_labels(pharo_model, comment, "pharo")
assert preds == expected
def test_python_invariance_typos_parameters(python_model, get_predicted_labels):
"""
Tests typo tolerance
"""
# Define the single expected outcome
expected_labels = {"Parameters"}
# Define the base case and all its variants (with typos)
variants = [
":param user_id: The ID of the user.",
":paramater user_id: The ID of the user.",
":pram user_id: The ID of teh user.",
]
# Loop through all variants and assert they all produce the *exact* expected outcome
for comment in variants:
preds = get_predicted_labels(python_model, comment, "python")
assert preds == expected_labels
def test_java_invariance_semantic_summary(java_model, get_predicted_labels):
"""
Tests semantic invariance
"""
# Get the prediction for the base comment
base_comment = "/** Returns the user ID. */"
base_preds = get_predicted_labels(java_model, base_comment, "java")
# Define semantic paraphrases of the base comment
variants = [
base_comment,
"/** Gets the user ID. */",
"/** Fetches the ID for the user. */",
"/** A method to return the user's ID. */",
]
# Check that the base prediction is valid (summary)
assert "summary" in base_preds
for comment in variants:
preds = get_predicted_labels(java_model, comment, "java")
assert preds == base_preds
|