#creating path
os.environ['PATH'] += r"C://Users/USER/Workspace"
#go to the website
driver = webdriver.Chrome()
driver.get(currenturls)
driver.implicitly_wait(20)
from bs4 import BeautifulSoup
import requests
for currenturl in currenturls:
html_text=requests.get(currenturls).text
soup= BeautifulSoup(html_text, 'lxml')
boxs=soup.find_all('a' , class_='list-group-item')
for box in boxs:
name=box.find('h5', class_='list-group-item-heading').text
num=box.find('p',class_='list-group-item-text').text.replace('EXISTING ','')
webpage=print(currenturls)
column1.append(name)
column2.append(num)
if i==0:
WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li/a")))
nextbutton=driver.find_element(By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li/a")
nextbutton.click()
i+=1
currenturls=driver.current_url
print('next')
else:
WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li[2]/a")))
nextbutton=driver.find_element(By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li[2]/a")
nextbutton.click()
currenturls=driver.current_url
print('next')
time.sleep(5)
result=pd.DataFrame(list(zip(column1,column2)),columns=['Company Name','Registeration Num'])
result.to_excel('draft2.xlsx',index=False)
My original URL is https://www.mysbusiness.com/browse/A. After I use Selenium to press next the advertisement pop up https://www.mysbusiness.com/browse/A/after/1111949/#google_vignette I try to manually close the advertisement, but then I realise the code skips the data for the page https://www.mysbusiness.com/browse/A/after/1111949/