12、外贸站采集代码

import scrapy from scrapy.crawler import CrawlerProcess import json lists = [] class ListSpider(scrapy.Spider): name = "list" start_urls = [ 'http://www.techstart.org.uk/index.php?main_page=advanced_search_result&search_in_description=1&keyword=a&inc_subcat=0&sort=20a&page=1', ] def parse(self, response): for index in response.css('h3.itemTitle'): data = { 'title': index.css('h3.itemTitle a::text').extract(), 'url': index.css('h3.itemTitle a::attr(href)').extract() } lists.append(data) # page = len(response.css('div.navSplitPagesLinks a::attr(href)')) # next_page = response.css('div.navSplitPagesLinks a::attr(href)')[page - 1].extract() # # if next_page is not None: # next_page = response.urljoin(next_page) # yield scrapy.Request(next_page, callback=self.parse) # with open('lists.json', 'w') as f: json.dump(lists, f) process = CrawlerProcess() process.crawl(ListSpider) process.start() import scrapy from scrapy.crawler import CrawlerProcess import json infos = [] class InfoSpider(scrapy.Spider): name = "info" # with open('lists.json', 'r') as f: # result = json.load(f) # start_urls = [str(k['url'])[2:-2] for k in result] start_urls = [ 'http://www.techstart.org.uk/24-kilates-x-11-by-boris-bidjan-saberi-x-reebok-insta-pump-fury-blackwhite-various-sizes-p-5768.html', ] def parse(self, response): for index in response.css('div.centerColumn'): data = { 'title': index.css('h1::text').extract(), } infos.append(data) # process = CrawlerProcess() # process.crawl(InfoSpider) # process.start() from bs4 import BeautifulSoup import requests urls = [] start_url = 'http://www.techstart.org.uk/index.php?main_page=advanced_search_result&search_in_description=1&keyword=a&inc_subcat=0&sort=20a&page=1' def get_list(url): response = requests.get(url) soup = BeautifulSoup(response.text,'lxml') titles = soup.select('h3.itemTitle') urls = soup.select('h3.itemTitle > a') for title,link in zip(titles,urls): data = { 'title':title.get_text(), 'url':link.get('href') } print(data)