OiO.lk Community platform!

Oio.lk is an excellent forum for developers, providing a wide range of resources, discussions, and support for those in the developer community. Join oio.lk today to connect with like-minded professionals, share insights, and stay updated on the latest trends and technologies in the development field.
  You need to log in or register to access the solved answers to this problem.
  • You have reached the maximum number of guest views allowed
  • Please register below to remove this limitation

Python Webscraper for Ebay is unavle to fetch results for multiple links

  • Thread starter Thread starter Shivanshu Dwivedi
  • Start date Start date
S

Shivanshu Dwivedi

Guest
The code is as:

Code:
import time
from bs4 import BeautifulSoup
import requests
import csv

def get_page(url):
    response = requests.get(url)

    if not response.ok:
        print("Server Responded: ", response.status_code)
    else:
        soup = BeautifulSoup(response.text, "lxml")

    return soup
    
def get_detail_data(soup):

    #title
    #price
    #imageURL

    try:
        title = soup.find("h1", class_="x-item-title__mainTitle").find("span").get_text().strip()
        print(title)

    except:
        title = ""

    try:
        price = soup.find("div", class_="x-price-primary").find("span").get_text().strip()
        print(price)

    except:
        price = ""

    try:
        image_url = soup.find("div", class_="ux-image-carousel-item image-treatment active image").find("img").get("src").strip()
        print(image_url)

    except:
        image_url = ""

    data = {
        'Title': title,
        'Price': price,
        'Image URL': image_url
    }

    return data

def get_index_data(soup):

    try:
        links = soup.find_all("a", class_="s-item__link").strip()
    except:
        links = []
    
    urls = [item.get("href") for item in links]

    print(urls)
    return urls
    

def main():

    
    url = "https://www.ebay.com/sch/i.html?_from=R40&_nkw=watch+men&_sacat=260324&_pgn=1"

    products = get_index_data(get_page(url))

    for link in products:
        print(link)
        data = get_detail_data(get_page(link))
        print(data)

if __name__ == "__main__":
    main()

I was trying to scrape watches section from Ebay from search functionality. It works individually for a single page of watch but when I try to seach using a for loop as done above. It returns the error :

Code:
Traceback (most recent call last):
  File "/Users/shivanshu/Documents/Projects/WebScrapers/Ebay/ebay_scraper.py", line 84, in <module>
    main()
  File "/Users/shivanshu/Documents/Projects/WebScrapers/Ebay/ebay_scraper.py", line 80, in main
    data = get_detail_data(get_page(link))
                           ^^^^^^^^^^^^^^
  File "/Users/shivanshu/Documents/Projects/WebScrapers/Ebay/ebay_scraper.py", line 14, in get_page
    return soup
           ^^^^
<p>The code is as:</p>
<pre><code>import time
from bs4 import BeautifulSoup
import requests
import csv

def get_page(url):
response = requests.get(url)

if not response.ok:
print("Server Responded: ", response.status_code)
else:
soup = BeautifulSoup(response.text, "lxml")

return soup

def get_detail_data(soup):

#title
#price
#imageURL

try:
title = soup.find("h1", class_="x-item-title__mainTitle").find("span").get_text().strip()
print(title)

except:
title = ""

try:
price = soup.find("div", class_="x-price-primary").find("span").get_text().strip()
print(price)

except:
price = ""

try:
image_url = soup.find("div", class_="ux-image-carousel-item image-treatment active image").find("img").get("src").strip()
print(image_url)

except:
image_url = ""

data = {
'Title': title,
'Price': price,
'Image URL': image_url
}

return data

def get_index_data(soup):

try:
links = soup.find_all("a", class_="s-item__link").strip()
except:
links = []

urls = [item.get("href") for item in links]

print(urls)
return urls


def main():


url = "https://www.ebay.com/sch/i.html?_from=R40&_nkw=watch+men&_sacat=260324&_pgn=1"

products = get_index_data(get_page(url))

for link in products:
print(link)
data = get_detail_data(get_page(link))
print(data)

if __name__ == "__main__":
main()

</code></pre>
<p>I was trying to scrape watches section from Ebay from search functionality. It works individually for a single page of watch but when I try to seach using a for loop as done above. It returns the error :</p>
<pre class="lang-none prettyprint-override"><code>Traceback (most recent call last):
File "/Users/shivanshu/Documents/Projects/WebScrapers/Ebay/ebay_scraper.py", line 84, in <module>
main()
File "/Users/shivanshu/Documents/Projects/WebScrapers/Ebay/ebay_scraper.py", line 80, in main
data = get_detail_data(get_page(link))
^^^^^^^^^^^^^^
File "/Users/shivanshu/Documents/Projects/WebScrapers/Ebay/ebay_scraper.py", line 14, in get_page
return soup
^^^^
</code></pre>
 

Latest posts

Top