WebMagic中使用POST请求并传递参数

Maven依赖

尽量使用最新版本的webmagic

<dependency> <groupId>us.codecraft</groupId> <artifactId>webmagic-core</artifactId> <version>0.6.1</version> </dependency> <dependency> <groupId>us.codecraft</groupId> <artifactId>webmagic-extension</artifactId> <version>0.6.1</version> </dependency>

发起POST请求

//设置POST请求 Request request = new Request("http://PostRequestUrl.com"); //只有POST请求才可以添加附加参数 request.setMethod(HttpConstant.Method.POST); //设置POST参数 List<NameValuePair> nvs = new ArrayList<NameValuePair>(); nvs.add(new BasicNameValuePair("key1", "value1")); nvs.add(new BasicNameValuePair("key2", "value2")); nvs.add(......); //转换为键值对数组 NameValuePair[] values = nvs.toArray(new NameValuePair[] {}); //将键值对数组添加到map中 Map<String, Object> params = new HashMap<String, Object>(); //key必须是：nameValuePair params.put("nameValuePair", values); //设置request参数 request.setExtras(params); // 开始执行 try { Spider.create(new MyPageProcessor()).addRequest(request).thread(5).run(); } catch (Exception e) { e.printStackTrace(); }

为什么参数map的key必须要用”nameValuePair”

类 us.codecraft.webmagic.downloader.HttpClientDownloader 中的selectRequestMethod(us.codecraft.webmagic.Request)方法

protected RequestBuilder selectRequestMethod(Request request) { String method = request.getMethod(); if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) { //default get return RequestBuilder.get(); } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) { //只有POST方法才会对request中的附加参数做处理 RequestBuilder requestBuilder = RequestBuilder.post(); //这里，从request中get时，key为：nameValuePair NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair"); if (nameValuePair != null && nameValuePair.length > 0) { requestBuilder.addParameters(nameValuePair); } return requestBuilder; } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) { return RequestBuilder.head(); } else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) { return RequestBuilder.put(); } else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) { return RequestBuilder.delete(); } else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) { return RequestBuilder.trace(); } throw new IllegalArgumentException("Illegal HTTP Method " + method); }