import pandas as pd
import sqlite3 as db
import requests

frames = []

s = '''
http://www.etf.com/channels/bond-etfs
http://www.etf.com/channels/mlp-etfs
http://www.etf.com/channels/silver-etfs
http://www.etf.com/channels/china-etfs
http://www.etf.com/channels/muni-etfs
http://www.etf.com/channels/us-broad-market-bond-etfs
http://www.etf.com/channels/dividend-etfs
http://www.etf.com/channels/natural-gas-etfs
http://www.etf.com/channels/global-bond-etfs
http://www.etf.com/channels/oil-etfs
http://www.etf.com/channels/gold-etfs
http://www.etf.com/channels/treasury-etfs
http://www.etf.com/channels/reit-etfs
http://www.etf.com/channels/high-dividend-yield-etfs
http://www.etf.com/channels/japan-etfs
http://www.etf.com/channels/smart-beta-etfs
http://www.etf.com/etf-lists/alternatives-etfs
http://www.etf.com/etf-lists/asset-allocation-etfs
http://www.etf.com/etf-lists/currency-etfs
http://www.etf.com/etf-lists/fixed-income-etfs
http://www.etf.com/channels/alpha-seeking-etfs
http://www.etf.com/channels/basic-materials-etfs
http://www.etf.com/channels/consumer-cyclicals-etfs
http://www.etf.com/channels/consumer-non-cyclicals-etfs
http://www.etf.com/channels/energy-etfs
http://www.etf.com/channels/extended-market-etfs
http://www.etf.com/channels/financials-etfs
http://www.etf.com/channels/health-care-etfs
http://www.etf.com/channels/high-dividend-yield-etfs
http://www.etf.com/channels/industrials-etfs
http://www.etf.com/channels/real-estate-etfs
http://www.etf.com/channels/small-cap-etfs
http://www.etf.com/channels/technology-etfs
http://www.etf.com/channels/telecommunications-etfs
http://www.etf.com/channels/theme-etfs
http://www.etf.com/channels/total-market-etfs
http://www.etf.com/channels/utilities-etfs
http://www.etf.com/channels/asia-pacific-etfs
http://www.etf.com/channels/developed-markets-etfs
http://www.etf.com/channels/emerging-markets-etfs
http://www.etf.com/channels/europe-etfs
http://www.etf.com/channels/global-etfs
http://www.etf.com/channels/global-ex-us-etfs
http://www.etf.com/channels/latin-america-etfs
http://www.etf.com/channels/middle-east-and-africa-etfs
'''
for i in s.split():
    df = pd.read_html(requests.get(i,headers={'User-agent': 'Mozilla/5.0'}).text)
    df[5].columns = ['Fund Name','Ticker','Asset Class','Region','Geography','Category','Focus',
                  'Niche', 'Inverse','Leveraged','ETN','Underlying Index','Selection Criteria',
                  'Weighting Scheme','Active per SEC','Index Provider','Strategy']
    frames.append(df[5])