This is the code use to connect to our hive database , it ran fine a week back but now it seems to be be failing to even open a session and get a cursor to execute the queries. The issue was temporarily fixed when i explicitly added a cursor.close() method but now it's back again. I am unable to access the hive database using python
I have tried using pyhs2 and pyhive both the libraries fail to connect to the hive database. Nothing on the cluster has changed so far. what could the reason for this be ?
I know Hive is not a relational DB so the concept of cursors doesn't make sense but Is there any way the the Hive database remembers the cursors created using the pyhive library ?? if so how can i delete the currently unused cursors ??
Here is the code and exception it raises upon execution
from pyhive import hive
import contextlib
class Hive():
def __init__(self,host="[hostnamehere]",db="default",port="10000",auth="KERBEROS",kerberos_service_name="hive"):
self.host = host
self.db = db
self.port = port
self.auth = auth
self.kerberos_service_name = kerberos_service_name
def connect(self):
return(hive.connect(host=self.host, port=self.port, database=self.db, auth=self.auth, kerberos_service_name=self.kerberos_service_name))
def query_one(self,sql):
with contextlib.closing(self.connect()) as connection:
with contextlib.closing(connection.cursor()) as cursor:
cursor.execute(sql)
result = cursor.fetch_one()
cursor.close()
return(result)
if __name__ == "__main__":
connector = Hive()
print("running query")
print(connector.query_one("SELECT * FROM [tablenamehere]"))
raise OperationalError(response) pyhive.exc.OperationalError: TOpenSessionResp(status=TStatus(statusCode=3, infoMessages=['*org.apache.hive.service.cli.HiveSQLException:Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient:13:12', 'org.apache.hive.service.cli.session.SessionManager:openSession:SessionManager.java:289', 'org.apache.hive.service.cli.CLIService:openSession:CLIService.java:199', 'org.apache.hive.service.cli.thrift.ThriftCLIService:getSessionHandle:ThriftCLIService.java:427', 'org.apache.hive.service.cli.thrift.ThriftCLIService:OpenSession:ThriftCLIService.java:319', 'org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession:getResult:TCLIService.java:1257', 'org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession:getResult:TCLIService.java:1242', 'org.apache.thrift.ProcessFunction:process:ProcessFunction.java:39', 'org.apache.thrift.TBaseProcessor:process:TBaseProcessor.java:39', 'org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingProcessor:process:HadoopThriftAuthBridge.java:562', 'org.apache.thrift.server.TThreadPoolServer$WorkerProcess:run:TThreadPoolServer.java:286', 'java.util.concurrent.ThreadPoolExecutor:runWorker:ThreadPoolExecutor.java:1149', 'java.util.concurrent.ThreadPoolExecutor$Worker:run:ThreadPoolExecutor.java:624', 'java.lang.Thread:run:Thread.java:748', '*java.lang.RuntimeException:java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient:15:2', 'org.apache.hadoop.hive.ql.session.SessionState:start:SessionState.java:547', 'org.apache.hive.service.cli.session.HiveSessionImpl:open:HiveSessionImpl.java:144', 'org.apache.hive.service.cli.session.SessionManager:openSession:SessionManager.java:281', '*java.lang.RuntimeException:Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient:21:6', 'org.apache.hadoop.hive.metastore.MetaStoreUtils:newInstance:MetaStoreUtils.java:1566', 'org.apache.hadoop.hive.metastore.RetryingMetaStoreClient::RetryingMetaStoreClient.java:92', 'org.apache.hadoop.hive.metastore.RetryingMetaStoreClient:getProxy:RetryingMetaStoreClient.java:138', 'org.apache.hadoop.hive.metastore.RetryingMetaStoreClient:getProxy:RetryingMetaStoreClient.java:110', 'org.apache.hadoop.hive.ql.metadata.Hive:createMetaStoreClient:Hive.java:3510', 'org.apache.hadoop.hive.ql.metadata.Hive:getMSC:Hive.java:3542', 'org.apache.hadoop.hive.ql.session.SessionState:start:SessionState.java:528', '*java.lang.reflect.InvocationTargetException:null:25:4', 'sun.reflect.NativeConstructorAccessorImpl:newInstance0:NativeConstructorAccessorImpl.java:-2', 'sun.reflect.NativeConstructorAccessorImpl:newInstance:NativeConstructorAccessorImpl.java:62', 'sun.reflect.DelegatingConstructorAccessorImpl:newInstance:DelegatingConstructorAccessorImpl.java:45', 'java.lang.reflect.Constructor:newInstance:Constructor.java:423', 'org.apache.hadoop.hive.metastore.MetaStoreUtils:newInstance:MetaStoreUtils.java:1564', '*org.apache.hadoop.hive.metastore.api.MetaException:GC overhead limit exceeded:30:4', 'org.apache.hadoop.hive.metastore.RetryingHMSHandler::RetryingHMSHandler.java:82', 'org.apache.hadoop.hive.metastore.RetryingHMSHandler:getProxy:RetryingHMSHandler.java:91', 'org.apache.hadoop.hive.metastore.HiveMetaStore:newRetryingHMSHandler:HiveMetaStore.java:6463', 'org.apache.hadoop.hive.metastore.HiveMetaStoreClient::HiveMetaStoreClient.java:206', 'org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient::SessionHiveMetaStoreClient.java:76', '*java.lang.OutOfMemoryError:GC overhead limit exceeded:0:-1'], sqlState=None, errorCode=0, errorMessage='Failed to open new session: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient'), serverProtocolVersion=7, sessionHandle=None, configuration=None)
