0

I would like to ask what I am doing wrong that I always select wrong item in dropdown list while webscrapping website. I am using Selenium, Pyhton. I need select item which contains (EUR). At this time it selects item SE not EUR as expected. My code part looks like this:

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import csv
import time
import glob
import os, fnmatch
import pandas as pd
import re, os.path
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from selenium.webdriver.firefox.options import Options as FirefoxOptions
#from msedge.selenium_tools import EdgeOptions
#from msedge.selenium_tools import Edge

print("Starting stock script")
p="C:/Users/apskaita3/Finansų analizės ir valdymo sprendimai, UAB/Rokas Toomsalu - Power BI analitika/Integracijos/1_Public comapnies analytics/2. Nasdaq Helsinki/Nasdaq_historical_prices"


stock_list =[


    {"name": "Nordea Bank Ab", "csv_name": "C:/Users/apskaita3/Finansų analizės ir valdymo sprendimai, UAB/Rokas Toomsalu - Power BI analitika/Integracijos/1_Public comapnies analytics/Databasesets/Nasdaq_historical_prices/NDA-FI.csv"}


 ]



for i in stock_list:
    csv_name = i['csv_name']
    print(f"Starting iteration {i}")
    cname = i['name']

    #chromeOptions = webdriver.ChromeOptions()
   
    #chromeOptions.add_argument('--remote-debugging-port=9230')
    
        
    #driver=webdriver.Firefox(executable_path=r'C:\Users\apskaita3\Desktop\gec\geckodriver.exe',service_log_path=os.devnull)
    #driver = webdriver.Edge(r"C:/Users/apskaita3/Desktop/msedgedriver.exe")

    """
    edge_options = EdgeOptions()
    edge_options.use_chromium = True
    edge_options.add_argument('headless')
    edge_options.add_argument("disable-gpu")
    edge_options.add_argument("--remote-debugging-port=9222")
    edge_options.add_argument("--no-sandbox")
    edge_options.add_argument("--disable-dev-shm-usage")
    """
    #driver = webdriver.Edge(r"C:/Users/apskaita3/Desktop/msedgedriver.exe",{"edge_options":"edge_options"})

    #driver = webdriver.Edge(r"C:/Users/apskaita3/Desktop/msedgedriver.exe",{"edge_options": edge_options})


    #driver = webdriver.Edge(r"C:/Users/apskaita3/Desktop/msedgedriver.exe", options=edge_options)



    #driver = webdriver.Firefox(executable_path=r'C:/Users/apskaita3/Documents/geckodriver.exe', service_log_path="C:/Users/apskaita3/Downloads/geckodriver.log")
    driver = webdriver.Chrome(r"C:/Users/apskaita3/Documents/chromedriver_win32/chromedriver.exe")


    
    if cname=="TietoEVRY Oy":



        #driver = webdriver.Edge(ChromeDriverManager().install())
        #driver = webdriver.Chrome("C:/Users/apskaita3/Desktop/chromedriver.exe")
        driver.maximize_window()
        driver.get("http://www.nasdaqomxnordic.com/shares/historicalprices")
        driver.refresh()
        fromDate = driver.find_element_by_id("FromDate")
        fromDate.click()
        fromDate.clear()
        fromDate.send_keys("2006-01-01")
        time.sleep(4)
        inputElement = driver.find_element_by_id("instSearchHistorical")
        inputElement.send_keys(cname)

        #driver.implicitly_wait(6)
        time.sleep(4)
        #s=driver.find_element_by_xpath("/html/body/ul[3]/li[3]/a/div[3]")
        #s=driver.find_elements_by_xpath("//*[contains(text(), 'EUR')]")
        #driver.find_elements_by_xpath("//div[contains(text(),'EUR')]")
        #s = driver.find_element_by_xpath("//div[.='" + foo + "']")
        #s=driver.find_element_by_xpath("//div[text()='My Button']")
        #s=driver.find_element_by_xpath("//div[normalize-space()='EUR']]")

        #driver.find_element_by_xpath("//a[@class='acSearchResult-fnm floatLeft']/div[contains(text(), '(EUR)')]/text()")

        driver.find_element_by_xpath("//a[@class='acSearchResult-fnm floatLeft']/div[contains(text(), '(EUR)')]/text()")


        a=driver.find_element_by_xpath("//div[contains(@class, 'acSearchResult-fnm floatLeft')]")
        print(a)
        driver.find_element_by_xpath("//div[contains(@class, 'acSearchResult-fnm floatLeft')]")

        #s=driver.find_element_by_xpath("//li[@class = 'ui-menu-item']/a/div[contains(text(), '(EUR)')]")

        #driver.find_element_by_xpath("//li[@class = 'ui-menu-item']/a/div[contains(text(), '(EUR)')]")
        print(s)
        
        print("driver")
        print(s)
        #s=driver.find_element_by_xpath("//*[text()='EUR']")
        #print("s",s)
        #s=driver.find_elements_by_xpath("//*[matches(.,'EUR', 'i')]")
        #element = wait.until(EC.visibility_of_element_located((By.XPATH, "//div[text()='Nordea Bank Abp (EUR)']")))
        #driver.execute_script("arguments[0].click();", s)
        #.click()
        #inputElement.send_keys(Keys.ENTER)

        #driver.find_element_by_css_selector('div.acSearchResult-fnm').click()


        new_csv1 = pd.read_csv(csv_name,sep=';')
        if ('sep='  in new_csv1.columns.tolist()):
            new_csv1= pd.read_csv(csv_name,sep=';',skiprows=1)
        else:
            new_csv1= pd.read_csv(csv_name,sep=',')

        new_df1 = pd.concat([old_csv1, new_csv1]).reset_index(drop=True)
        #old_csv1=new_csv1
        new_df1.columns = old_csv1.columns
        new_df1['Trades']=new_df1['Trades'].replace(" ", "")
        new_df1=new_df1.sort_values('Date', ascending = False).drop_duplicates()
        if 'Unnamed: 11' in new_df1.columns.tolist():
            del new_df1['Unnamed: 11']
        else:
            pass
        new_df1.to_csv(csv_name,index = False)
        print(new_df1.duplicated())


        os.chdir("C:/Users/apskaita3/Downloads")
        for e in os.listdir():
          if "-2006-01-01" in e:
            os.remove(e)
          else:
            pass
                

        
        filenames.clear()
        time.sleep(3)       

        driver.find_element_by_id("showAdjusted").click()

                        
        time.sleep(4)
        driver.find_element_by_id("exportExcel").click()
        print(5)
        time.sleep(8)
        
        driver.close()

        time.sleep(3)
        print(5)

        #path = "C:/Users/apskaita3/Downloads/"
        accepted_extensions = ["csv"]
        filenames = [fn for fn in os.listdir("C:/Users/apskaita3/Downloads/") if fn.split(".")[-1] in accepted_extensions]
        #filenames = sorted(filenames, key=lambda t: -os.stat(t).st_mtime)
        #print(filenames)
        #filenames=re.search("2006",filenames) 
        #print(filenames)
         
        filenames = [s for s in filenames if "-2006-01-01" in s]

        print(os.listdir("C:/Users/apskaita3/Downloads/"))
        
        print(filenames)
        a1=filenames[0]

        try:
            old_csv1 = pd.read_csv(f'C:/Users/apskaita3/Downloads/{a1}',sep=';',skiprows=1)
        except pd.errors.EmptyDataError:
          #old_csv14 = pd.DataFrame()
            old_csv1  = pd.DataFrame(columns=['Date', 'Bid', 'Ask', 'Opening price','High price','Low price','Closing price', 'Average price','Total volume','Turnover','Trades'])

        new_csv1 = pd.read_csv(csv_name,sep=';')
        if ('sep='  in new_csv1.columns.tolist()):
            new_csv1= pd.read_csv(csv_name,sep=';',skiprows=1)
        else:
            new_csv1= pd.read_csv(csv_name,sep=',')

        new_df1 = pd.concat([old_csv1, new_csv1]).reset_index(drop=True)
        #old_csv1=new_csv1
        new_df1.columns = old_csv1.columns
        new_df1['Trades']=new_df1['Trades'].replace(" ", "")
        new_df1=new_df1.sort_values('Date', ascending = False).drop_duplicates()
        if 'Unnamed: 11' in new_df1.columns.tolist():
            del new_df1['Unnamed: 11']
        else:
            pass
        new_df1.to_csv(csv_name,index = False)
        print(new_df1.duplicated())


        os.chdir("C:/Users/apskaita3/Downloads")
        for e in os.listdir():
          if "-2006-01-01" in e:
            os.remove(e)
          else:
            pass
                

        
        filenames.clear()
        time.sleep(3)
    else:
        #driver = webdriver.Edge(ChromeDriverManager().install())
        #driver = webdriver.Chrome("C:/Users/apskaita3/Desktop/chromedriver.exe")
        driver.maximize_window()
        driver.get("http://www.nasdaqomxnordic.com/shares/historicalprices")
        driver.refresh()
        fromDate = driver.find_element_by_id("FromDate")
        fromDate.click()
        fromDate.clear()
        fromDate.send_keys("2006-01-01")
        time.sleep(4)
        inputElement = driver.find_element_by_id("instSearchHistorical")
        inputElement.send_keys(cname)

        #driver.implicitly_wait(6)
        time.sleep(4)
        s=driver.find_element_by_xpath("/html/body/ul[2]/li[2]/a/div[1]")
        driver.execute_script("arguments[0].click();", s)
        #.click()
        #inputElement.send_keys(Keys.ENTER)

        #driver.find_element_by_css_selector('div.acSearchResult-fnm').click()

        time.sleep(2)

        driver.find_element_by_id("showAdjusted").click()

                        
        time.sleep(4)
        driver.find_element_by_id("exportExcel").click()
        print(5)
        time.sleep(8)
        
        driver.close()

        time.sleep(3)
        print(5)

        #path = "C:/Users/apskaita3/Downloads/"
        accepted_extensions = ["csv"]
        filenames = [fn for fn in os.listdir("C:/Users/apskaita3/Downloads/") if fn.split(".")[-1] in accepted_extensions]
        #filenames = sorted(filenames, key=lambda t: -os.stat(t).st_mtime)
        #print(filenames)
        #filenames=re.search("2006",filenames) 
        #print(filenames)
         
        filenames = [s for s in filenames if "-2006-01-01" in s]

        print(os.listdir("C:/Users/apskaita3/Downloads/"))
        
        print(filenames)
        a1=filenames[0]

        try:
            old_csv1 = pd.read_csv(f'C:/Users/apskaita3/Downloads/{a1}',sep=';',skiprows=1)
        except pd.errors.EmptyDataError:
          #old_csv14 = pd.DataFrame()
            old_csv1  = pd.DataFrame(columns=['Date', 'Bid', 'Ask', 'Opening price','High price','Low price','Closing price', 'Average price','Total volume','Turnover','Trades'])

        new_csv1 = pd.read_csv(csv_name,sep=';')
        if ('sep='  in new_csv1.columns.tolist()):
            new_csv1= pd.read_csv(csv_name,sep=';',skiprows=1)
        else:
            new_csv1= pd.read_csv(csv_name,sep=',')

        new_df1 = pd.concat([old_csv1, new_csv1]).reset_index(drop=True)
        #old_csv1=new_csv1
        new_df1.columns = old_csv1.columns
        new_df1['Trades']=new_df1['Trades'].replace(" ", "")
        new_df1=new_df1.sort_values('Date', ascending = False).drop_duplicates()
        if 'Unnamed: 11' in new_df1.columns.tolist():
            del new_df1['Unnamed: 11']
        else:
            pass
        new_df1.to_csv(csv_name,index = False)
        print(new_df1.duplicated())


        os.chdir("C:/Users/apskaita3/Downloads")
        for e in os.listdir():
          if "-2006-01-01" in e:
            os.remove(e)
          else:
            pass
                

        
        filenames.clear()
        time.sleep(3)
     

enter image description here

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Browse other questions tagged or ask your own question.