Htykuut

I would like to download images off https://www.pixiv.net/, so it was a little troublesome. Had to log in to even begin scraping details off pages. When I tried to urllib.request.urlretrieve them I get a 403 forbidden error. I searched the web for other methods but they always end up with a 403 forbidden error

Here's a sample of the page i want to scrape,
https://www.pixiv.net/member_illust.php?mode=medium&illust_id=71751889

To even begin scraping, one would have to log in, you wouldn't be able to find the necessary elements without logging in.

import requests

import time

import urllib.request

from bs4 import BeautifulSoup

from selenium import webdriver

from selenium.webdriver.common.keys import Keys

from selenium.webdriver.support.ui import Select



def login(browser):

    Log_In = browser.find_element_by_link_text('Login')

    Log_In.click()

    Username = browser.find_element_by_xpath("//*[@id='LoginComponent']/form/div[1]/div[1]/input")

    Username.send_keys('') #input username

    Password = browser.find_element_by_xpath("//*[@id='LoginComponent']/form/div[1]/div[2]/input")

    Password.send_keys('') #input password

    Login = browser.find_elements_by_tag_name('button')[1]

    time.sleep(1)

    Login.click()





def search(browser):

    time.sleep(1)

    searchbox = browser.find_element_by_id('suggest-input')

    searchbox.send_keys('toyosatomimi no miko')

    searchbox.send_keys(Keys.ENTER)

    image = browser.find_element_by_class_name('_25taFA4')

    image.click()





def get_soup(browser):

    return BeautifulSoup(browser.page_source, 'lxml')



def download_image(soup, file_path):

    url = soup.find_all('a', {'target': '_blank'})[1].get('href')

    file_name = 'image'

    full_path = file_path + file_name + '.jpg'

    urllib.request.urlretrieve(url,full_path)



url = "https://www.pixiv.net/"

browser = webdriver.Chrome(r'D:\chromedriver_win32\chromedriver.exe')

browser.get(url)

login(browser)

search(browser)

soup = get_soup(browser)

browser.get(url)

soup = get_soup(browser)

download_image(soup, 'D:\instagram_photos')





Traceback (most recent call last):

 File "D:/pixiv scraper/venv/pixiv scrape.py", line 95, in <module>

    download_image(soup, 'D:\instagram_photos')

  File "D:/pixiv scraper/venv/pixiv scrape.py", line 57, in download_image

    urllib.request.urlretrieve(url,full_path)

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

   32liburllibrequest.py", line 247, in urlretrieve

    with contextlib.closing(urlopen(url, data)) as fp:

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

   32liburllibrequest.py", line 222, in urlopen

    return opener.open(url, data, timeout)

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

   32liburllibrequest.py", line 531, in open

    response = meth(req, response)

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

    32liburllibrequest.py", line 641, in http_response

    'http', request, response, code, msg, hdrs)

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

   32liburllibrequest.py", line 569, in error

    return self._call_chain(*args)

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

   32liburllibrequest.py", line 503, in _call_chain

    result = func(*args)

  File "C:UsersHPAppDataLocalProgramsPythonPython37- 

    32liburllibrequest.py", line 649, in http_error_default

    raise HTTPError(req.full_url, code, msg, hdrs, fp)

    urllib.error.HTTPError: HTTP Error 403: Forbidden

This is my code, i wonder if anyone can help?

.urlretrieve() has no cookies or session like in selenium browser that why you got 403, and you also need to set user-agent.

def download_image(browser, file_path):

    userAgent = browser.execute_script("return navigator.userAgent;")

    seleniumCookies= browser.get_cookies()

    cookies = ''

    for cookie in seleniumCookies:

        cookies += '%s=%s;' % (cookie['name'], cookie['value'])



    opener = urllib.request.build_opener()

    opener.addheaders = [('User-Agent', userAgent)]

    opener.addheaders.append(('Cookie', cookies))



    soup = get_soup(browser)

    url = soup.find_all('a', {'target': '_blank'})[1].get('href')

    file_name = 'image'

    full_path = file_path + file_name + '.jpg'

    urllib.request.urlretrieve(url,full_path)



url = "https://www.pixiv.net/"

browser = webdriver.Chrome(r'D:\chromedriver_win32\chromedriver.exe')

browser.get(url)

login(browser)

search(browser)

# you may need to WebDriverWait until search result appear

download_image(browser, 'D:\instagram_photos')