elencocapitalia/scraper.py at master · massimomagris/elencocapitalia · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import scraperwiki
import lxml.html
import BeautifulSoup
from BeautifulSoup import BeautifulSoup

def cerca(html,search,nrline,offset):
    i=html.find(search)
    #print "I " + str(i)
    if i > 0:
        i=i+len(search)
        j=i+offset
        #esegue split ed elimina tutto quello che inizia con <
        s = html[i:j]
        s = s.splitlines(nrline)
        #print "S " + str(s)
        rs = s[nrline]
        rs = rs.strip ("\r")
        rs = rs.strip ("\n")
        return (rs)
    else:
        return ''

pagine = range(1,97)

for count in pagine:
    try:
        searchURL = "http://www.comuni-italiani.it/cap/%02d.html"%count
        print searchURL
        html = scraperwiki.scrape(searchURL)
        html = BeautifulSoup(html,convertEntities=BeautifulSoup.HTML_ENTITIES)
        tables = html.findAll('table')

    except:
        print "error for WKI_ID:"+ str(id)

    rows = tables[5].findAll('tr')
    li = {}

    for row in rows:
        try:
            cells = row.findChildren('td')
            if len(cells) <= 4:
                li['CAP'] = cells[0].text
                li['COMUNE'] = cells[1].text
                #print cells
                #print cells[1]
                #print str(cells[1])[27:40]
                comune =  str(cells[1])[31:39]
                comuneUrl = "http://www.comuni-italiani.it/" + comune
                #print comuneUrl

                html = scraperwiki.scrape(comuneUrl)
                #print html
                print cerca (html,'../../tel/',0,10)
                #html = BeautifulSoup(html,convertEntities=BeautifulSoup.HTML_ENTITIES)
                #Trovare il modo di recuperare il prefisso del comune e altri dati

                #match = re.search(r'href=?+)', str(cells[1]))
                #if match:
                #    print match.group(0)


                scraperwiki.sqlite.save(unique_keys=['CAP','COMUNE'], data=li)
        except:
            print "error for WKI_ID:"+ str(id)