I am trying to do some scraping from websites using GET and POST methods, but now I am facing a new challenge.
I am trying to get data from a credit simulator, I found this portuguese site (https://simuladores.bancomontepio.pt/ITSCredit.External/Calculator/ITSCredit.Calculator.UI.External/gateway/Calculator/api/Calculator/Calculate?hash=-1535840118).
As far as I know, I need to use POST method,but I am only getting 401 for the response.
Here is my code:
Any ideas on what the problem is?
from openpyxl import load_workbook
import requests
import numpy as np
import time
from datetime import datetime
import pandas as pd
import json
import warnings
warnings.filterwarnings("ignore")
#O MONTANTE MINIMO A SER FINANCIADO NO MONTEPIO É 2500€
mont_min = 2000
mont_max = 30000
mont_step = 500
prazo_min = 36
prazo_max = 96
prazo_step = 6
response_errors = []
def benchmark(PRAZO, MONTANTE):
# colunas da tabela final
columns = ['date','amount','monthly_payment_without_ins','MTIC', 'maturity', 'taeg', 'tan', 'type']
# bancos analisados
df = pd.DataFrame(np.nan, index=['Montepio'], columns=columns)
pd.set_option('display.max_columns', 10)
# aux_MONTANTE = (str('{:0,.0f}'.format(MONTANTE)) + " €").replace(",", ".")
# aux_PRAZO = str(PRAZO) + " meses"
url_Montepio = 'https://simuladores.bancomontepio.pt/ITSCredit.External/Calculator/ITSCredit.Calculator.UI.External/gateway/Calculator/api/Calculator/Calculate?hash=-1535840118'
headers_Montepio = {'Accept': 'application/json, text/plain, */*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language':'pt-PT,pt;q=0.9,en-US;q=0.8,en;q=0.7',
'Connection': 'keep-alive',
'Content-Length': '969',
'content-type': 'text/plain',
'Cookie': 'OptanonAlertBoxClosed=2023-02-15T13:57:43.951Z; _gcl_au=1.1.776232285.1676469471; ASP.NET_SessionId=5ggffmgwygzrqhda0ufmdgip; _ga=GA1.2.829667105.1676469478; _gid=GA1.2.888620949.1678890412; calc-cookie=; _ga_8WVEJF7X11=GS1.1.1678890400.2.0.1678890496.60.0.0; _gat=1; OptanonConsent=isGpcEnabled=0&datestamp=Wed+Mar+15+2023+14%3A28%3A18+GMT%2B0000+(Hora+padr%C3%A3o+da+Europa+Ocidental)&version=6.38.0&isIABGlobal=false&hosts=&consentId=a56ebf5e-f207-48ad-90a4-746316bbace0&interactionCount=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0002%3A0%2CC0003%3A0%2CC0004%3A0&geolocation=ES%3BMD&AwaitingReconsent=false; _gat_UA-186811106-6=1; _gali=sliderAmount',
'Host': 'simuladores.bancomontepio.pt',
'Origin': 'https://simuladores.bancomontepio.pt',
'Referer': 'https://simuladores.bancomontepio.pt/',
'sec-ch-ua': '"Google Chrome";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'}
payload_Montepio = {"CCRDCalculateInput":{},"MultifunctionsCalculateInput":{},
"Device":
{"Browser":"chrome",
"BrowserVersion":"107.0.0.0",
"Device":"Desktop",
"Os":"windows",
"OsVersion":"windows-10",
"UserAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36"},
"IsCustomer":"true",
"Ammount":3500,
"Term":36,
"ConditionCode":"26BU199902X-01-I-199902-F",
"CreditDestinationCode":"199902",
"ProductCode":"26BU199902X",
"FinancedExpenses":"false",
"FrequencyTrancheCode":"null",
"GoalCode":"E001",
"GoalDescription":"EQUIPAMENTO ENERGIAS RENOVÁVEIS",
"FrequencyTypeCode":"M",
"FamilyCode":"CP",
"Proponents":
[{"Position":1,
"Birthday":"1993-03-15T13:28:18.000Z",
"State":"true",
"EntityType"
:{"ID":1,
"CompanyID":1,
"Code":"P",
"Description":"Proponente",
"Value":"null",
"ValueString":"null",
"State":"true",
"Imported":"null"},
"ExpenseCodes":["009"]}],
"ResidualValue":0,
"InterestOnly":0,
"Deferment":0,
"OptionalExpenses":
[{"Code":"009",
"Factor":1}],
"Counterparts":0}
response_Montepio = requests.post(url_Montepio, headers=headers_Montepio, data=payload_Montepio, verify=False).json()
Any ideas on what the problem is?