I am trying to scroll down a drop-down menu, but since scrolling with a mouse will just scroll the page instead of the drop-down, I thought of using sendkeys by sending DOWN_ARROW multiple times. However this doesn't work very well/is pretty inconsistent.
Does anyone have a solution? I need to scroll down in order to find the element's path. Trying to extract from all the dates from the first "Dates" dropdown menu here. Currently can only do so for the latest 14 https://www.sgx.com/research-education/securities Anything beyond that python reports that it's unable to find the element...
for i in range(10):
dropdowndate.click()
driver.implicitly_wait(5)
selectdate = driver.find_element_by_xpath('//*[@data-key='+str(i)+']')
dates.append(selectdate.get_attribute('title'))
webdriver.ActionChains(driver).move_to_element(selectdate).perform()
selectdate.click()
driver.implicitly_wait(5)
driver.execute_script("arguments[0].click();", DownloadButton)
driver.implicitly_wait(20)
time.sleep(5)
#11th to 14th elements
for j in range(10,20):
dropdowndate.click()
driver.implicitly_wait(5)
webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN * j).perform()
#dropdowndate.send_keys(Keys.ARROW_DOWN*(13))
driver.implicitly_wait(30)
selectdate = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, '//*[@data-key='+str(j)+']')))
dates.append(selectdate.get_attribute('title'))
driver.implicitly_wait(10)
webdriver.ActionChains(driver).move_to_element(selectdate).perform()
driver.implicitly_wait(10)
selectdate.click()
driver.implicitly_wait(5)
driver.execute_script("arguments[0].click();", DownloadButton)
driver.implicitly_wait(10)
Full code is here:
from selenium import webdriver #for navigating through the pages
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
from bs4 import BeautifulSoup #webscraping fundamentals
import urllib.request
import pandas as pd #Data manipulation
import re
import os
import json #to save in multiple folder for looping
from pandas import Series, ExcelWriter #write output to excel in different tab
from selenium.common.exceptions import NoSuchElementException
options = webdriver.ChromeOptions() #selecting the webdriver
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
options.add_argument('--headless')
driver = webdriver.Chrome(executable_path=r'C:\\Users\\user\\Downloads\\chromedriver.exe') #to always download and update accordingly
#If prompted error on version: go to your browser, type in 'chrome://version' and check version.
#If eg, Google Chrome 80.0.3987.132, go to 'https://chromedriver.chromium.org/downloads' to download
#the same version and update the link above accordingly. Note that browser version should be same as driver version
URL = "https://www.sgx.com/research-education/securities"
driver.get(URL)
driver.implicitly_wait(5) #asking machine to wait 5 seconds before performing other action
#manually block pop-up if any before running next
response = driver.page_source
soup = BeautifulSoup(response,"lxml")
#CLICK ACCEPT COOKIES
Cookies = driver.find_element_by_xpath('//*[@id="gdpr-banner"]/div/button')
Cookies.click()
driver.implicitly_wait(5)
#TYPE OF DATA
dropdowntype = driver.find_element_by_name('type')
dropdowntype.click()
driver.implicitly_wait(3)
selecttype = driver.find_element_by_xpath('//*[@id="sgx-select-dialog"]/div[2]/sgx-select-picker\
/sgx-list/div/div/sgx-select-picker-option[1]/label/span')
#current path is for price, change option[1] to option [2] and option [3] for trade amendment and price adjustment respectively
selecttype.click()
#DATES
dates = []
dropdowndate = driver.find_element_by_name('date')
DownloadButton=driver.find_element_by_xpath('//*[@id="page-container"]/template-base/div/div/section[1]/div/sgx-widgets-wrapper\
/widget-research-and-reports-download[1]/widget-reports-securities/div/button')
#1st to 10th elements
for i in range(10):
dropdowndate.click()
driver.implicitly_wait(5)
selectdate = driver.find_element_by_xpath('//*[@data-key='+str(i)+']')
dates.append(selectdate.get_attribute('title'))
webdriver.ActionChains(driver).move_to_element(selectdate).perform()
selectdate.click()
driver.implicitly_wait(5)
driver.execute_script("arguments[0].click();", DownloadButton)
driver.implicitly_wait(20)
time.sleep(5)
#11th to 14th elements
for j in range(10,20):
dropdowndate.click()
driver.implicitly_wait(5)
webdriver.ActionChains(driver).send_keys(Keys.ARROW_DOWN * j).perform()
#dropdowndate.send_keys(Keys.ARROW_DOWN*(13))
driver.implicitly_wait(30)
selectdate = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, '//*[@data-key='+str(j)+']')))
dates.append(selectdate.get_attribute('title'))
driver.implicitly_wait(10)
webdriver.ActionChains(driver).move_to_element(selectdate).perform()
driver.implicitly_wait(10)
selectdate.click()
driver.implicitly_wait(5)
driver.execute_script("arguments[0].click();", DownloadButton)
driver.implicitly_wait(10)
