Unable to log in to a website using the requests module

Question

I’m trying to log in to a website using requests module. It seems I have incorporated the manual steps into the script based on what I see in dev tools while logging in to that site manually. However, when I run the script and check the content it received as a response, I see this line: There was an unexpected error.

I’ve created a free account there for the purpose of testing only. The login details are hardcoded within the parameters.

import requests
from bs4 import BeautifulSoup

link = 'https://www.apartments.com/customers/login'
login_url = 'https://auth.apartments.com/login?{}'
params = {
    'dsrv.xsrf': '',
    'sessionId': '',
    'username': 'email',
    'password': 'password'
}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
}

headers_post = {
    'origin': 'https://auth.apartments.com',
    'referer': '',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'en-US,en;q=0.9,bn;q=0.8',
    'x-requested-with': 'XMLHttpRequest'
}
with requests.Session() as s:
    s.headers.update(headers)
    resp = s.get(link)
    soup = BeautifulSoup(resp.text,"lxml")
    res = s.get(soup.select_one("#auth-signin-iframe")['src'])
    soup = BeautifulSoup(res.text,"lxml")
    post_url = login_url.format(soup.select_one("[id='signinform']")['action'].split("/login?")[1])
    headers_post['referer'] = post_url
    s.headers.update(headers_post)
    params['dsrv.xsrf'] = soup.select_one("input[name='idsrv.xsrf']")['value']
    params['sessionId'] = soup.select_one("input[id='sessionId']")['value']
    resp = s.post(post_url,data=params)
    print(resp.status_code)
    print(resp.content)
    print(resp.url)

How can I make the login successful using the requests module?

Asked By: robots.txt

||

Source

Answer 1

You are missing an i in your params. its idsrv.xsrf not dsrv.xsrf. Here is the fixed version:

import requests
import lxml
from bs4 import BeautifulSoup

link = 'https://www.apartments.com/customers/login'
login_url = 'https://auth.apartments.com/login?{}'
params = {
    'idsrv.xsrf': '',
    'sessionId': '',
    'username': 'USERNAME',
    'password': 'PASSWORD'
}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
}

headers_post = {
    'origin': 'https://auth.apartments.com',
    'referer': '',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'en-US,en;q=0.9,bn;q=0.8',
    'x-requested-with': 'XMLHttpRequest',
    'Sec-Fetch-Dest': 'iframe',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1'
}
with requests.Session() as s:
    s.headers.update(headers)
    resp = s.get(link)
    soup = BeautifulSoup(resp.text,"lxml")
    res = s.get(soup.select_one("#auth-signin-iframe")['src'])
    soup = BeautifulSoup(res.text,"lxml")
    post_url = login_url.format(soup.select_one("[id='signinform']")['action'].split("/login?")[1])
    headers_post['referer'] = post_url
    s.headers.update(headers_post)
    params['idsrv.xsrf'] = soup.find('input', {'name': 'idsrv.xsrf'}).get('value')
    params['sessionId'] = soup.find('input', {'id': 'sessionId'}).get('value')
    resp = s.post(post_url,data=params)
    print(resp.status_code)
    print(resp.content)
    print(resp.url)

Please note, in order to let it run in my environment, I had to change the way the values are read from the soup.

Also, I removed your user credentials, for safety 🙂

Answered By: Lukas Hestermeyer

Answer 2

I was poking around the website and got this piece to work:

with requests.Session() as session:
    r = session.get("https://www.apartments.com/customers/login", headers={
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/110.0",
    })
    soup = BeautifulSoup(r.text)
    link = soup.select_one("#auth-signin-iframe")['src']

    r = session.get(link, headers={
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/110.0",
    })
    soup = BeautifulSoup(r.text)
    login_url = "https://auth.apartments.com" + soup.select_one("#signinform")["action"]
    data = {
        "idsrv.xsrf": soup.select_one("[name='idsrv.xsrf']")["value"],
        "sessionId": soup.select_one("#sessionId").get("value", ""),
        "username": EMAIL,
        "password": PASSWORD,
    }
    r = session.post(login_url, data=data, headers={
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/110.0",
    })
    soup = BeautifulSoup(r.text)
    data = {
        "code": soup.select_one("[name='code']")["value"],
        "state": soup.select_one("[name='state']")["value"],
        "session_state": soup.select_one("[name='session_state']")["value"],
    }
    r = session.post("https://www.apartments.com/customers/Authentication/CompleteSignIn", data=data, headers={
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/110.0",
    })

    # Use the above response object for your use case

Note the use of the Session object to utilize the cookies that get communicated to the script.

Answered By: bleuatb

Unable to log in to a website using the requests module

Question:

Answers: