i am trying to fetch 500 cars data for each of the 11 cities whose urls are saved in url_city variable from car dekho.com, but each time, i m getting this exception error. please suggest me some better way to get rid of this.
code block for function car_data(city):
def car_data(city):
reg_year=[]
fuel=[]
kms=[]
eng_power=[]
owner=[]
automation=[]
ins=[]
variant=[]
brand=[]
mod_name=[]
colour=[]
rto=[]
price=[]
href_tags=[]
url=[]
while len(href_tags)<500:
href_tags=set(href_tags)
href_tags=driver.find_elements('xpath',"//div[@class='gsc_col-xs-7 carsName']/a")
driver.execute_script("window.scrollBy(0,document.body.scrollHeight)")
time.sleep(0.5)
continue
for q in href_tags :
url.append(q.get_attribute('href'))
print(len(url))
for r in url:
driver.get(r)
delay=30
try:
b=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[2]/div/div[2])[1]")
except NoSuchElementException:
b=driver.find_element('xpath',"(//div[@class='iconDetail'])[8]")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
b=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[2]/div/div[2])[1]")
if b.text=='':
reg_year.append(np.NaN)
else:
reg_year.append(b.text)
try:
c=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[3]/div/div[2])[1]")
except NoSuchElementException:
c=driver.find_element('xpath',"(//div[@class='UCcontCard GenDetails posR gsc_row']/div[1]/ul/li[5]/div/div)")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
c=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[3]/div/div[2])[1]")
if c.text=='':
fuel.append(np.NaN)
else:
fuel.append(c.text)
try:
d= driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[4]/div/div[2])[1]")
except NoSuchElementException:
d= driver.find_element('xpath',"(//div[@class='UCcontCard GenDetails posR gsc_row']/div[1]/ul/li[3]/div/div)")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
d= driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[4]/div/div[2])[1]")
if d.text=='':
kms.append(np.NaN)
else:
kms.append(d.text.split(' ')[0])
try:
e= driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[5]/div/div[2])[1]")
except NoSuchElementException:
e= driver.find_element('xpath',"//div[@class='contentwrap gsc_col-xs-12']/section/div/div/ul/li[2]/div[2]")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
e= driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[5]/div/div[2])[1]")
if e.text=='':
eng_power.append(np.NaN)
else:
eng_power.append(e.text.split(' ')[0])
try:
f=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[6]/div/div[2])[1]")
except NoSuchElementException:
f=driver.find_element('xpath',"(//div[@class='UCcontCard GenDetails posR gsc_row']/div[1]/ul/li[4]/div/div)")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
f=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[6]/div/div[2])[1]")
if f.text == '':
owner.append(np.NaN)
else:
owner.append(f.text.split(' ')[0])
try:
g=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[8]/div/div[2])[1]")
except NoSuchElementException:
g=driver.find_element('xpath',"(//div[@class='UCcontCard GenDetails posR gsc_row']/div[1]/ul/li[6]/div/div)")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
g=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[8]/div/div[2])[1]")
if g.text == '':
automation.append(np.NaN)
else:
automation.append(g.text)
try:
h = driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[9]/div/div[2])[1]")
except NoSuchElementException:
h = driver.find_element('xpath',"(//div[@class='UCcontCard GenDetails posR gsc_row']/div[1]/ul/li[7]/div/div)")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
h = driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[9]/div/div[2])[1]")
if h.text=='':
ins.append(np.NaN)
else:
ins.append(h.text)
try:
i= driver.find_element('xpath',"//div[@class='paddingBorder clearfix']/div[1]/div[1]")
except NoSuchElementException:
i1= driver.find_element('xpath',"//div[@class='vdp-head']/h1[2]")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'paddingBorder clearfix')))
i= driver.find_element('xpath',"//div[@class='paddingBorder clearfix']/div[1]/div[1]")
try:
if i.text=='':
variant.append(np.NaN)
else :
variant.append(i.text)
except StaleElementReferenceException:
if i1.text=='':
variant.append(np.NaN)
else :
variant.append(i1.text.split(' ')[-1])
try:
j = driver.find_element('xpath',"//div[@class='paddingBorder clearfix']/div[1]/h1")
except NoSuchElementException:
j = driver.find_element('xpath',"//div[@class='vdp-head']/h1[2]")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'paddingBorder clearfix')))
j = driver.find_element('xpath',"//div[@class='paddingBorder clearfix']/div[1]/h1")
if j.text.split(' ')[1] == '':
brand.append(np.NaN)
else:
brand.append(j.text.split(' ')[1])
if j.text.split(' ')[2] == '':
mod_name.append(np.NaN)
else:
mod_name.append(j.text.split(' ')[2:4])
try:
k= driver.find_element('xpath',"(//div[@class='white-bg VDPtopCard overviewCArd']/div[1]/div[1]/ul/li[6]/div/div[2])[2]")
except NoSuchElementException:
k= driver.find_element('xpath',"//div[@class='contentwrap gsc_col-xs-12']/section/div/div/ul/li[7]/div[2]")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'white-bg VDPtopCard overviewCArd')))
k= driver.find_element('xpath',"(//div[@class='white-bg VDPtopCard overviewCArd']/div[1]/div[1]/ul/li[6]/div/div[2])[2]")
if k.text == '':
colour.append(np.NaN)
else:
colour.append(k.text)
try:
l=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[7]/div/div[2])[1]")
except NoSuchElementException:
l=driver.find_element('xpath',"(//div[@class='UCcontCard GenDetails posR gsc_row']/div[1]/ul/li[8]/div/div)")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
l=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[7]/div/div[2])[1]")
if l.text == '':
rto.append(np.NaN)
else:
rto.append(l.text)
try:
m= driver.find_element('xpath',"//div[@class='paddingBorder clearfix']/div[2]/div[1]/div/span[2]")
except NoSuchElementException:
m= driver.find_element('xpath',"(//span[@class='amount'])[1]")
except StaleElementReferenceException:
WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'paddingBorder clearfix')))
m= driver.find_element('xpath',"//div[@class='paddingBorder clearfix']/div[2]/div[1]/div/span[2]")
if m.text == '':
price.append(np.Nan)
else:
price.append(m.text)
print(f"for city_url {city}, the following is the length of each columns")
print(len(reg_year),len(fuel),len(kms),len(eng_power),len(owner),len(automation))
print(len(ins),len(variant),len(brand),len(mod_name),len(colour),len(rto), len(price))
df1=pd.DataFrame({'Reg_year':reg_year,'Fuel_used':fuel,'Kms_driven':kms,'Engine_power':eng_power,'No_of_owners':owner,'Automation':automation,'Insurance':ins,'Brand_name':brand,'Model_name':mod_name,'Type_of_variant':variant,'Colour':colour,'Rto':rto,'Price':price})
df=pd.concat([df,df1],ignore_index=True)
print(df)
and the final execution is done by using the code:
for city in city_urls:
driver.get(city)
time.sleep(5)
car_data(city)
time.sleep(5)
it then saves the href for 500 cars and then start transitions from first car up to sometime 125 number of cars, sometimes 178 number of cars etc. (each time up to different number of cars), then show this error. the error i m getting each time is:
520
---------------------------------------------------------------------------
StaleElementReferenceException Traceback (most recent call last)
Input In [9], in <cell line: 1>()
2 driver.get(city)
3 time.sleep(5)
----> 4 car_data(city)
5 time.sleep(5)
Input In [8], in car_data(city)
34 WebDriverWait(driver,delay).until(EC.presence_of_element_located((By.CLASSNAME,'gsc_row detailsList')))
35 b=driver.find_element('xpath',"(//ul[@class='gsc_row detailsList']/li[2]/div/div[2])[1]")
---> 37 if b.text=='':
38 reg_year.append(np.NaN)
39 else:
File ~\Anaconda_setup\lib\site-packages\selenium\webdriver\remote\webelement.py:84, in WebElement.text(self)
81 @property
82 def text(self) -> str:
83 """The text of the element."""
---> 84 return self._execute(Command.GET_ELEMENT_TEXT)['value']
File ~\Anaconda_setup\lib\site-packages\selenium\webdriver\remote\webelement.py:396, in WebElement._execute(self, command, params)
394 params = {}
395 params['id'] = self._id
--> 396 return self._parent.execute(command, params)
File ~\Anaconda_setup\lib\site-packages\selenium\webdriver\remote\webdriver.py:428, in WebDriver.execute(self, driver_command, params)
426 response = self.command_executor.execute(driver_command, params)
427 if response:
--> 428 self.error_handler.check_response(response)
429 response['value'] = self._unwrap_value(
430 response.get('value', None))
431 return response
File ~\Anaconda_setup\lib\site-packages\selenium\webdriver\remote\errorhandler.py:243, in ErrorHandler.check_response(self, response)
241 alert_text = value['alert'].get('text')
242 raise exception_class(message, screen, stacktrace, alert_text) # type: ignore[call-arg] # mypy is not smart enough here
--> 243 raise exception_class(message, screen, stacktrace)
StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=104.0.5112.102)
Stacktrace:
Backtrace:
Ordinal0 [0x011178B3+2193587]
Ordinal0 [0x010B0681+1771137]
Ordinal0 [0x00FC41A8+803240]
Ordinal0 [0x00FC6BB4+814004]
Ordinal0 [0x00FC6A72+813682]
Ordinal0 [0x00FC6D00+814336]
Ordinal0 [0x00FEDF7F+974719]
Ordinal0 [0x0100D7AC+1103788]
Ordinal0 [0x00FE8C04+953348]
Ordinal0 [0x0100D9C4+1104324]
Ordinal0 [0x0101DAE2+1170146]
Ordinal0 [0x0100D5C6+1103302]
Ordinal0 [0x00FE77E0+948192]
Ordinal0 [0x00FE86E6+952038]
GetHandleVerifier [0x013C0CB2+2738370]
GetHandleVerifier [0x013B21B8+2678216]
GetHandleVerifier [0x011A17AA+512954]
GetHandleVerifier [0x011A0856+509030]
Ordinal0 [0x010B743B+1799227]
Ordinal0 [0x010BBB68+1817448]
Ordinal0 [0x010BBC55+1817685]
Ordinal0 [0x010C5230+1856048]
BaseThreadInitThunk [0x766A6739+25]
RtlGetFullPathName_UEx [0x772690AF+1215]
RtlGetFullPathName_UEx [0x7726907D+1165]
source https://stackoverflow.com/questions/73487689/can-anybody-please-tell-me-why-this-stale-element-reference-exception-is-arising
Comments
Post a Comment