GeezerVison

Since #2583 fuzzer is able to construct quite complex arguments for method under test, that can take quite some time to actually create, while instrumnted-process marks executions as timed out even if the thing that actually timed out is the argument creation prior to method under test invocation.

To Reproduce

Use UtBot version from Introduce object creation using builders for fuzzers #2583
Generate tests with fuzzing set to 100% for the following method

public static boolean isNull(CharSequence charSequence) {
    if (charSequence == null) return true;
    else return false;
}

Expected behavior

Two successful test and no time out test.

Actual behavior

Two successful test and 10 time out test.

Visual proofs

public final class ClassUnderTestTest {
    ///region Test suites for executable org.example.ClassUnderTest.isNull

    ///region FUZZER: SUCCESSFUL EXECUTIONS for method isNull(java.lang.CharSequence)

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = null -> return true")
    public void testIsNullReturnsTrue() {
        boolean actual = ClassUnderTest.isNull(null);

        assertTrue(actual);
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = '10\uFFFC' (mutated from '10') -> return false")
    public void testIsNullReturnsFalseWithNonEmptyString() {
        boolean actual = ClassUnderTest.isNull("10\uFFFC");

        assertFalse(actual);
    }
    ///endregion

    ///region FUZZER: TIMEOUTS for method isNull(java.lang.CharSequence)

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        byte[] byteArray1 = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        inetAddress.getByAddress(byteArray1);
        inetAddress.getLocalHost();
        inetAddress.getByName("");
        byte[] byteArray2 = {Byte.MIN_VALUE, (byte) 0, (byte) -1};
        inetAddress.getByAddress("-3", byteArray2);
        inetAddress.getLoopbackAddress();
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull1() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        byte[] byteArray1 = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        inetAddress.getByAddress(byteArray1);
        inetAddress.getLocalHost();
        inetAddress.getByName("");
        byte[] byteArray2 = {Byte.MIN_VALUE, (byte) 0, (byte) -1};
        inetAddress.getByAddress("-3", byteArray2);
        inetAddress.getLoopbackAddress();
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull2() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        inetAddress.getByName("");
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull3() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull4() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull5() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull6() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull7() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull8() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }

    /**
     * @utbot.classUnderTest {@link ClassUnderTest}
     * @utbot.methodUnderTest {@link ClassUnderTest#isNull(CharSequence)}
     */
    @Test
    @DisplayName("isNull: arg_0 = InetAddress.getCanonicalHostName()")
    @Timeout(value = 1000L, unit = TimeUnit.MILLISECONDS)
    public void testIsNull9() throws UnknownHostException {
        byte[] byteArray = {Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE, Byte.MIN_VALUE};
        InetAddress inetAddress = getByAddress(byteArray);
        String charSequence = inetAddress.getCanonicalHostName();
        
        /* This execution may take longer than the 1000 ms timeout
         and therefore fail due to exceeding the timeout. */
        assertTimeoutPreemptively(Duration.ofMillis(1000L), () -> ClassUnderTest.isNull(charSequence));
    }
    ///endregion

    ///endregion
}

Activity

IlyaMuravjov
added
ctg-bugIssue is a bugIssue is a bug

spec-regressionRegressionRegression

comp-fuzzingIssue is related to the fuzzingIssue is related to the fuzzing

comp-instrumented-processIssue is related to Instrumented processIssue is related to Instrumented process
on Sep 12, 2023
korifey
added this to UTBot Javaon Sep 12, 2023
github-project-automation
moved this to Todo in UTBot Javaon Sep 12, 2023
alisevych commented on Sep 19, 2023
alisevych
on Sep 19, 2023
Member
More actions
Probably related issue:

Timeout tests are sometimes generated by Fuzzing for simple functions #2262
olganaumenko
added a commit that references this issue on Oct 13, 2023
Save already created mock model in models to variables cache #2494 #2595
ca7df7c

Metadata

Assignees

No one assigned

Labels

Type

No type

Projects

UTBot Java
Status
Todo
Show more project fields

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions

You are visitor 01402503 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.

GeezerVison

Multiple time out tests are generated by fuzzer #2595

Description

Activity

alisevych commented on Sep 19, 2023

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions