You could check if a file with the given basename exists and otherwise rename the file with the smallest number appended to the basename. Try something like this:
Const basename = "test"
Const srcFolder = "..."
Const extension = "txt"
Set fso = CreateObject("Scripting.FileSystemObject")
dstFile = fso.BuildPath(srcFolder, basename & "." & extension)
If fso.FileExists(dstFile) Then WScript.Quit 0 'nothing to do
For Each f In fso.GetFolder(srcFolder).Files
If LCase(fso.GetExtensionName(f.Name)) = extension Then
If LCase(Left(f.Name, Len(basename))) = basename Then
num = Mid(fso.GetBaseName(f.Name), Len(basename)+1)
If Len(num) > 0 Then
num = CInt(num)
If IsEmpty(minnum) Or minnum > num Then minnum = num
End If
End If
End If
Next
If Not IsEmpty(minnum) Then
srcFile = fso.BuildPath(srcFolder, basename & minnum & "." & extension)
fso.MoveFile srcFile, dstFile
End If
The checks on the file name and the number could be simplified a little by testing against a regular expression:
Set re = New RegExp
re.Pattern = "^" & basename & "(\d+)\." & extension & "$"
re.IgnoreCase = True
For Each f In fso.GetFolder(srcFolder).Files
Set m = re.Execute(f.Name)
If m.Count > 0 Then
num = CInt(m(0).SubMatches(0))
If IsEmpty(minnum) Or minnum > num Then minnum = num
End If
Next