testthat::auto_test() seems to fail when S4 code is involved

Question

Over the last couple of weeks, I've been integrating the unit-test functionality of the testthat package more and more into my daily work and really have to say: this package rocks!

I specifically like the autotest-feature which will automatically run all of your unit tests every time you make a change somewhere in your code. However, I think I came across a little bug or at least some undesired behavior when using testthat's auto_test() function in situations when your code directory contains S4 definitions (classes, generic methods, custom methods).

Question

Is this behavior illustrated below a bug or am I doing something wrong when using auto_test()?

Below you'll find an illustration that requires to create some example files first

Preliminaries

Set working directory

setwd("testthat_autotest")

Ensure directories

dir.create("src", showWarnings=FALSE)
dir.create("tests", showWarnings=FALSE)

Create S4 Reference Class Definition

Creates file classes.R in directory src

def <- c(
    "setRefClass(",
    "   Class=\"MyClass\",",
    "   fields=list(",
    "       name=\"character\"",
    "   ),",
    "   methods=list(",
    "       getName=function(...) {",
    "           getName_ref(.self=.self, ...)",
    "       },",
    "       setName=function(...) {",
    "           setName_ref(.self=.self, ...)",
    "       }",
    "   )",
    ")"
)
write(def, file="src/classes.R")

Create Generic Method Definitions

Creates file generics.R in directory src

def <- c(
    "setGeneric(",
    "    name=\"getName_ref\",",
    "    signature=c(\".self\"),",
    "    def=function(",
    "        .self,",
    "        ...",
    "    ) {",
    "standardGeneric(\"getName_ref\")",    
    "   }",
    ")",
    "",
    "setGeneric(",
    "   name=\"setName_ref\",",
    "   signature=c(\".self\"),",
    "   def=function(",
    "       .self,",
    "       value,",
    "       ...",
    "   ) {",
    "   standardGeneric(\"setName_ref\")",    
    "   }",
    ")"
)
write(def, file="src/generics.R")

Create Custom Method Definitions

Creates file methods.R in directory src

def <- c(
    "require(\"compiler\")",
    "",
    "setMethod(",
    "   f=\"getName_ref\",", 
    "   signature=signature(.self=\"MyClass\"),", 
    "   definition=cmpfun(function(",
    "       .self,",
    "       ...",
        ") {",
        "out <- .self$name",
        "return(out)",
        "}, options=list(suppressAll=TRUE))",  
    ")",
    "setMethod(",
    "   f=\"setName_ref\",", 
    "   signature=signature(.self=\"MyClass\"),", 
    "   definition=cmpfun(function(",
    "       .self,",
    "       ...",
    "   ) {",
    "   .self$field(name=\"name\", value=value)",
    "   return(TRUE)",
    "   }, options=list(suppressAll=TRUE))",  
    ")"
)
write(def, file="src/methods.R")

Create Unit Tests

Creates files test_getName_ref.R and test_setName_ref.R in directory tests

test <- c(
    "require(\"testthat\")",
    "",
    "test_that(desc=\"test_getName_ref,.self=MyClass\",",
    "   code={",
    "       x.target <- \"some value\"",
    "       expect_that(",
    "            .self <- new(\"MyClass\", name=x.target),",
    "            is_a(\"MyClass\")",
    "        )",
    "        expect_that(",
    "            x.test <- .self$getName(),",
    "            is_a(\"character\")",
    "        )",
    "        expect_that(",
    "            x.test,",
    "            is_identical_to(x.target)",
    "        )",
    "    }",
    ")"
)
write(test, file="tests/test_getName_ref.R")

test <- c(
    "require(\"testthat\")",
    "",
    "test_that(desc=\"test_setName_ref,.self=MyClass\",",
    "   code={",
    "       x.target <- \"some value\"",
    "       expect_that(",
    "            .self <- new(\"MyClass\"),",
    "            is_a(\"MyClass\")",
    "        )",
    "        expect_that(",
    "            .self$setName(value=x.target),",
    "            is_identical_to(TRUE)",
    "        )",
    "        expect_that(",
    "            .self$getName(),",
    "            is_identical_to(x.target)",
    "        )",
    "    }",
    ")"
)
write(test, file="tests/test_setName_ref.R")

Illustration

Running auto_test() will result in errors:

require("testthat")

auto_test(
    code_path="src",
    test_path="tests"
)
Loading required package: compiler
.1.2

1. Error: test_getName_ref,.self=MyClass ---------------------------------------
could not find function "getName_ref"
1: expect_that(x.test <- .self$getName(), is_a("character"))
2: condition(object)
3: str_c(class(x), collapse = ", ")
4: Filter(function(x) length(x) > 0, list(...))
5: sapply(x, f)
6: lapply(X = X, FUN = FUN, ...)
7: .self$getName()

2. Error: test_setName_ref,.self=MyClass ---------------------------------------
could not find function "setName_ref"
1: expect_that(.self$setName(value = x.target), is_identical_to(TRUE))
2: condition(object)
3: all.equal(expected, actual)
4: all.equal.default(expected, actual)
5: all.equal.raw(target, current, ...)
6: attr.all.equal(target, current, ...)
7: mode(current)
8: .self$setName(value = x.target)

My first guess is that it might have something to do with the the way testthat::source_dir() handles the sourcing, maybe due to argument env?

In order to continue, we need to turn off autotesting with CRTL + BREAK (on Windows; see section "Autotest" in this article if you're on other platforms)

Now, when we first source the code manually and run test_dir(), everything seems to work fine:

require("testthat")

sapply(list.files("src", full.names=TRUE), source)

test_dir("tests")

......
You are a coding rockstar! 
>