I think of your solution like this:
for each subtree of the tree:
if the subtree is a binary search tree:
compute its size
if it is the largest one found so far:
best = subtree
return best
This is inefficient because it does O(n) work for each subtree, and there are up to n subtrees.
You can do better by walking the whole tree only once.
// Walk the subtree at node. Find the largest subtree that is a binary search tree
// and return that tree in *result. Also return that subtree's size and the range
// of values it covers in *size, *min, and *max.
void
walk(Node *node, Node **result, size_t *size, Value *min, Value *max)
{
Node *result0 = NULL;
size_t size0 = 0;
Value min0, max0;
if (node->left)
walk(node->left, &result0, &size0, &min0, &max0);
Node *result1 = NULL;
size_t size1 = 0;
Value min1, max1;
if (node->right)
walk(node->right, &result1, &size1, &min1, &max1);
// If both subtrees are search trees and node->value falls between them,
// then node is a search tree.
if (result0 == node->left
&& result1 == node->right
&& (node->left == NULL || max0 <= node->value)
&& (node->right == NULL || node->value <= min1))
{
*result = node;
*size = size0 + 1 + size1;
*min = node->left == NULL ? node->value : min0;
*max = node->right == NULL ? node->value : max1;
} else if (size0 >= size1) {
*result = result0;
*size = size0;
*min = min0;
*max = max0;
} else {
*result = result1;
*size = size1;
*min = min1;
*max = max1;
}
}
Node *
findLargestBinarySearchSubtree(Node *root)
{
Node *result;
size_t size;
Value min, max;
walk(root, &result, &size, &min, &max);
return result;
}