Python - Scrap data from tabular sheet and insert into sqlite3 database.

#Scrap data from tabular sheet and insert into sqlite3 database.

#listing the inserted records to console.

from bs4 import BeautifulSoup

import sys

import requests

from datetime import date

import sqlite3

now = date.today()

startdt = sys.argv[1]

enddt = sys.argv[2]

url = 'coinmarketcap.com/currencies/bitcoin/historical-data/?start='+startdt+'&end='+enddt+''

r  = requests.get("https://" +url)

print(url)

data = r.text

soup = BeautifulSoup(data,"lxml")

table = soup.find("table", { "class" : "table" })

currencytype = soup.find("select",{"class" : "pointer"})

row=table.findAll('tr')

row1 = table.findAll('li')

records = []

count = 0

def writedb():            

    conn = sqlite3.connect('bitcoin.db')

    cursor = conn.cursor()

    #Drop table if already exists in the database.

    cursor.execute('DROP TABLE IF EXISTS bitcoin')

    #create table

    cursor.execute('''CREATE TABLE bitcoin

                 (Date text, Open text, High text, Low text, Close text, Volume text,MarketCap text)''')

    conn.commit()

    #reading tabular data

    for row in table.findAll("tr")[1:]:

        cells = row.findAll("td")

        Date = cells[0].find(text=True)

        Open = cells[1].find(text=True)

        High = cells[2].find(text=True)

        Low = cells[3].find(text=True)

        Close = cells[4].find(text=True)

        Volume = cells[5].find(text=True)

        MarketCap = cells[6].find(text=True)

        record = (Date,Open,High,Low,Close,Volume,MarketCap)

        #insert data to table

        cursor.execute('INSERT INTO bitcoin VALUES (?,?,?,?,?,?,?)', record )

   

    #listing data from tables.

    cursor.execute("SELECT * FROM bitcoin") 

    result = cursor.fetchall() 

    

    for r in result:

        print(r)

                

    print("No of Records:%s"%len(result))    

#calling function

writedb();

#executing the script in command prompt

python e:\learning\python\beautifulsoup_sqlite.py 20160101 20171106

#output