1
sqoop import  --connect 'jdbc:sybase:Tds:10.100.*.***:5500/DATABASE=****' --driver 'com.sybase.jdbc3.jdbc.SybDriver' --username "keswara" --password "****" --target-dir "/user/keswara/WT_CONSUMERSTATS" --verbose --query "select c.YEARMONTH as d_stat_yearmonth,b.CONSNO,a.CONSCURRJDGMNTNUM as d_conscurrjdgmntnum,a.CONSCURRDFLTNUM as d_conscurrdfltnum,a.CONSCURRNOTICENUM as d_conscurrnoticenum,a.CONSCURRNOTRLBONDNUM as d_conscurrnotrlbondnum,a.CONSCURRDFLTADMINNUM as d_conscurrdfltadminnum,a.AMNISTYIND as d_amnisty_ind,a.NCRCREDITACTIVE as d_ncr_credit_active_ind,b.ACTIVEIND as d_active_ind,a.ESTINCOME as d_income,'d_create_date' as d_create_date,e.SUBURBCODE as d_physaddrsuburb_code,
    a.ENQHARDTOTALNUM as d_enqhardtotalnum,a.ENQSOFTTOTALNUM as d_enqsofttotalnum,a.ENQHARDNUMMONTH as d_enqhardnummonth,a.ENQSOFTNUMMONTH as d_enqsoftnummonth,a.ENQTOTALNUM as d_enqtotalnum,a.ENQTOTALNUMMONTH as d_enqtotalnummonth,'d_cug' as d_cug,a.CCACRWORSTEVER as d_crworstever,a.CCAINSWORSTEVER as d_insworstever,a.CCACRWORSTEVER as d_worstever,a.CCANUMACC as d_numacc,a.CCANUMOPENACC as d_numopenacc,a.CCANUMCLOSEDACC as d_numclosedacc,a.CCANUMNEGCLOSEDACC as  d_numnegclosedacc,a.CCANUMPOSCLOSEDACC as d_numposclosedacc,a.CCANUMACTIVEACC as d_numactiveacc,a.CCANUMWRITEOFFS as d_numwriteoffs,a.CCANUMDECEASEDWRITEOFFS as d_numdeceasedwriteoffs,
a.CCANUMHANDEDOVER as d_numhandedover,a.CCANUMCRCARDREVOKE as d_numcrcardrevoke,a.CCANUMREPO as d_numrepo,a.CCATOTALINSTALMENTAMT as d_totalinstalmentamt,a.CCATOTALOPENBAL as d_totalopenbal,a.CCATOTALCURRBAL as d_totalcurrbal,a.CCATOTALOVDUECRAMT as d_totalovduecramt,a.CCATOTALOVDUEDRAMT as d_totalovduedramt,a.CCANUMPAIDUPDEFACC as d_numpaidupdefacc,a.CCANUMSUSPENDEDACC as d_numsuspendedacc,a.CCANUMFROZENACC as d_numfrozenacc,d.SCORE as d_con_no,d.EXCLUSIONCODE as d_exclusion_code,'d_score_date' as d_score_date from dw.FT_CONSUMERSTATS a 
inner join dw.DM_CONSUMER as b on a.CONSKEY = b.CONSKEY,inner join dw.DM_MONTH as c  on c.MONTHKEY = a.MONTHKEY,inner join dw.FT_CONSUMER_SCORE as d on d.CONSKEY = b.CONSKEY,inner join dw.DM_SUBURB as e on b.SUBURBKEY = e.SUBURBKEY where yearmonth = 201501 AND \$CONDITIONS" --split-by 1 --verbose --create-hive-table --hive-table wt_consumer_stats --hive-import --map-column-hive d_stat_yearmonth=integer,d_con_no=integer,d_conscurrjdgmntnum=integer,d_conscurrdfltnum=integer,d_conscurrnoticenum=integer,d_conscurrnotrlbondnum=integer,d_conscurrdfltadminnum=integer,d_amnisty_ind=string,d_ncr_credit_active_ind=string,d_active_ind=string,d_income=integer,d_create_date=datetime,d_physaddrsuburb_code=integer,d_enqhardtotalnum=integer,d_enqsofttotalnum=integer,
d_enqhardnummonth=integer,d_enqsoftnummonth=integer,d_enqtotalnum=integer,d_enqtotalnummonth=integer,d_cug=string,d_crworstever=string,d_insworstever=string,d_worstever=string,d_numacc=integer,d_numopenacc=integer,d_numclosedacc=integer,d_numnegclosedacc=integer,d_numposclosedacc=integer,d_numactiveacc=integer,d_numwriteoffs=integer,d_numdeceasedwriteoffs=integer,d_numhandedover=integer,d_numcrcardrevoke=integer,d_numrepo=integer,d_totalinstalmentamt=decimal,d_totalopenbal=decimal,d_totalcurrbal=decimal,d_totalovduecramt=decimal,d_totalovduedramt=decimal,d_numpaidupdefacc=integer,d_numsuspendedacc=integer,d_numfrozenacc=integer,d_score_version=string,d_score=integer,d_exclusion_code=string,d_score_date=integer
franklinsijo
  • 17,784
  • 4
  • 45
  • 63
karthee
  • 49
  • 2
  • 14
  • 1
    What is the question here? Also post only the relevant section of the command here... – franklinsijo Mar 23 '17 at 16:39
  • When i run this script...am getting ''Query must contain '$CONDITIONS' in where clause"!!! That was the question! :) – karthee Mar 24 '17 at 06:34
  • But @karthee your `sqoop` command already contain `$CONDITIONS` in it. Do you still get the error or need an explanation why it is needed? – franklinsijo Mar 24 '17 at 06:37
  • 1
    @franklinsijo Yeah!! that's the problem here! Am still getting the error! even though script contains $CONDITIONS!! – karthee Mar 24 '17 at 07:35
  • From sqoop documentation : The facility of using free-form query in the current version of Sqoop is limited to simple queries where there are no ambiguous projections and no OR conditions in the WHERE clause. Use of complex queries such as queries that have sub-queries or joins leading to ambiguous projections can lead to unexpected results. May be try to simply query or create a temp table with complex query first and then use that to import instead. – Pushkr Mar 24 '17 at 15:45
  • @Pushkr that's correct! it may lead to un expected results! but the script should run! – karthee Mar 27 '17 at 07:48
  • can you try `where \$CONDITIONS and yearmonth = 201501'` instead of reverse what you have in code. – Pushkr Mar 27 '17 at 14:56
  • @Pushkr sure...let me try that! thanks again! – karthee Mar 28 '17 at 11:54

2 Answers2

1

did you try something like this?

sqoop import --connect 'jdbc:sybase:Tds:10.100.*.***:5500/DATABASE=****' --driver 'com.sybase.jdbc3.jdbc.SybDriver' --username "keswara" --password "****" --target-dir "/user/keswara/WT_CONSUMERSTATS" --verbose --query "select * from ( select c.YEARMONTH as d_stat_yearmonth,b.CONSNO,a.CONSCURRJDGMNTNUM as d_conscurrjdgmntnum,a.CONSCURRDFLTNUM as d_conscurrdfltnum,a.CONSCURRNOTICENUM as d_conscurrnoticenum,a.CONSCURRNOTRLBONDNUM as d_conscurrnotrlbondnum,a.CONSCURRDFLTADMINNUM as d_conscurrdfltadminnum,a.AMNISTYIND as d_amnisty_ind,a.NCRCREDITACTIVE as d_ncr_credit_active_ind,b.ACTIVEIND as d_active_ind,a.ESTINCOME as d_income,'d_create_date' as d_create_date,e.SUBURBCODE as d_physaddrsuburb_code, a.ENQHARDTOTALNUM as d_enqhardtotalnum,a.ENQSOFTTOTALNUM as d_enqsofttotalnum,a.ENQHARDNUMMONTH as d_enqhardnummonth,a.ENQSOFTNUMMONTH as d_enqsoftnummonth,a.ENQTOTALNUM as d_enqtotalnum,a.ENQTOTALNUMMONTH as d_enqtotalnummonth,'d_cug' as d_cug,a.CCACRWORSTEVER as d_crworstever,a.CCAINSWORSTEVER as d_insworstever,a.CCACRWORSTEVER as d_worstever,a.CCANUMACC as d_numacc,a.CCANUMOPENACC as d_numopenacc,a.CCANUMCLOSEDACC as d_numclosedacc,a.CCANUMNEGCLOSEDACC as d_numnegclosedacc,a.CCANUMPOSCLOSEDACC as d_numposclosedacc,a.CCANUMACTIVEACC as d_numactiveacc,a.CCANUMWRITEOFFS as d_numwriteoffs,a.CCANUMDECEASEDWRITEOFFS as d_numdeceasedwriteoffs, a.CCANUMHANDEDOVER as d_numhandedover,a.CCANUMCRCARDREVOKE as d_numcrcardrevoke,a.CCANUMREPO as d_numrepo,a.CCATOTALINSTALMENTAMT as d_totalinstalmentamt,a.CCATOTALOPENBAL as d_totalopenbal,a.CCATOTALCURRBAL as d_totalcurrbal,a.CCATOTALOVDUECRAMT as d_totalovduecramt,a.CCATOTALOVDUEDRAMT as d_totalovduedramt,a.CCANUMPAIDUPDEFACC as d_numpaidupdefacc,a.CCANUMSUSPENDEDACC as d_numsuspendedacc,a.CCANUMFROZENACC as d_numfrozenacc,d.SCORE as d_con_no,d.EXCLUSIONCODE as d_exclusion_code,'d_score_date' as d_score_date from dw.FT_CONSUMERSTATS a inner join dw.DM_CONSUMER as b on a.CONSKEY = b.CONSKEY,inner join dw.DM_MONTH as c on c.MONTHKEY = a.MONTHKEY,inner join dw.FT_CONSUMER_SCORE as d on d.CONSKEY = b.CONSKEY,inner join dw.DM_SUBURB as e on b.SUBURBKEY = e.SUBURBKEY where yearmonth = 201501 ) t10 where \$CONDITIONS" --split-by 1 --verbose --create-hive-table --hive-table wt_consumer_stats --hive-import --map-column-hive d_stat_yearmonth=integer,d_con_no=integer,d_conscurrjdgmntnum=integer,d_conscurrdfltnum=integer,d_conscurrnoticenum=integer,d_conscurrnotrlbondnum=integer,d_conscurrdfltadminnum=integer,d_amnisty_ind=string,d_ncr_credit_active_ind=string,d_active_ind=string,d_income=integer,d_create_date=datetime,d_physaddrsuburb_code=integer,d_enqhardtotalnum=integer,d_enqsofttotalnum=integer, d_enqhardnummonth=integer,d_enqsoftnummonth=integer,d_enqtotalnum=integer,d_enqtotalnummonth=integer,d_cug=string,d_crworstever=string,d_insworstever=string,d_worstever=string,d_numacc=integer,d_numopenacc=integer,d_numclosedacc=integer,d_numnegclosedacc=integer,d_numposclosedacc=integer,d_numactiveacc=integer,d_numwriteoffs=integer,d_numdeceasedwriteoffs=integer,d_numhandedover=integer,d_numcrcardrevoke=integer,d_numrepo=integer,d_totalinstalmentamt=decimal,d_totalopenbal=decimal,d_totalcurrbal=decimal,d_totalovduecramt=decimal,d_totalovduedramt=decimal,d_numpaidupdefacc=integer,d_numsuspendedacc=integer,d_numfrozenacc=integer,d_score_version=string,d_score=integer,d_exclusion_code=string,d_score_date=integer

hlagos
  • 7,690
  • 3
  • 23
  • 41
1

in the --query ".... where \$CONDITIONS AND yearmonth = 201501"

Marco Forte
  • 31
  • 2