[Python]代码    

import re  import urllib  import urllib.request  from collections import deque    queue = deque()#存放待爬取的网址  visited = set()#存放爬取过的网址。判断是否爬取过    url = "http://news.dbanotes.net"#入口网站  queue.append(url)  count = 1    while queue:   url = queue.popleft()#删除已经爬取过的队首的网址url   visited |= {url}#把已经爬取过的页面放入set中，方便下面的判断   urlop = urllib.request.urlopen(url)   if 'html' not in urlop.getheader('Content-Type'):    continue#如果是html再继续爬取   try:    data = urlop.read().decode('utf-8')   except:    continue   value = re.findall(r'href="(.+?)"',data)   for x in value:    if 'http' in x and x not in visited:     print("加入队列：" + x)