0
#creating path
os.environ['PATH'] += r"C://Users/USER/Workspace"

#go to the website
driver = webdriver.Chrome()
driver.get(currenturls)
driver.implicitly_wait(20)

from bs4 import BeautifulSoup
import requests



for currenturl in currenturls:
        html_text=requests.get(currenturls).text
        soup= BeautifulSoup(html_text, 'lxml')
        boxs=soup.find_all('a' , class_='list-group-item')

        
    for box in boxs:
        name=box.find('h5', class_='list-group-item-heading').text
        num=box.find('p',class_='list-group-item-text').text.replace('EXISTING ','')
        webpage=print(currenturls)
        column1.append(name)
        column2.append(num)
        

    if i==0:
        WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li/a")))
        nextbutton=driver.find_element(By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li/a")
        nextbutton.click()
        i+=1
        currenturls=driver.current_url
        print('next')

    
    else:
        WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li[2]/a")))
        nextbutton=driver.find_element(By.XPATH,"/html/body/div[1]/div/div[3]/div/div[3]/div[2]/nav/ul/li[2]/a")
        nextbutton.click()
        currenturls=driver.current_url
        print('next')
        time.sleep(5)


result=pd.DataFrame(list(zip(column1,column2)),columns=['Company Name','Registeration Num'])

result.to_excel('draft2.xlsx',index=False)

My original URL is https://www.mysbusiness.com/browse/A. After I use Selenium to press next the advertisement pop up https://www.mysbusiness.com/browse/A/after/1111949/#google_vignette I try to manually close the advertisement, but then I realise the code skips the data for the page https://www.mysbusiness.com/browse/A/after/1111949/

3
  • 1
    Ideally you don't. It's more efficient to "close" such popups by injecting cookies into the browser before you start executing your checks.
    – pavelsaman
    Jan 24 at 9:47
  • Thanks for reply. What do you mean by injecting cookies into browser?
    – bbq
    Jan 25 at 3:14
  • Browser has its own API for Cookie management, you can use it to set cookies. If the popup is managed by a cookie, you can set the cookie before loading the site. developer.mozilla.org/en-US/docs/web/api/document/cookie
    – pavelsaman
    Jan 25 at 8:44

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.