1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
| var config = [{
type: "waitFor",
selector: "body .WB_miniblog .PCD_followlist",
timeout: 30
}, {
type: 'script',
async: true,
timeout: 60,
func: function(args, target, callback) {
new Promise(function(f) {
if (window.jQuery && window.$) return f(0);
var script = document.createElement("script");
script.type = "text/javascript";
script.onload = f;
script.src = "//cdn.bootcss.com/jquery/2.1.4/jquery.min.js";
document.body.appendChild(script);
}).then(() => {
var html = $('body .WB_miniblog');
function transform(t) {
var r = [];
t.find('dd.mod_info').each((i, v) => {
var o = {},
$t = $(v);
o.name = $t.find('.info_name a strong').text();
o.url = $t.find('.info_name a').attr('href');
o.connect = $t.find('.info_connect span:eq(0) .count').text();
o.fans = $t.find('.info_connect span:eq(1) .count').text();
o.weibo = $t.find('.info_connect span:eq(2) .count').text();
o.address = $t.find('.info_add span').text();
o.intro = $t.find('.info_intro span').text();
r.push(o)
});
return r;
}
target.data = transform(html);
target.cat = args.cat;
callback();
});
}
}
]
let pool = new WebPagePool(5);
let callback = data => {
if (!result[data.cat])result[data.cat]=[];
result[data.cat].push(...data.data);
}
let result={};
let cats =
[{"url":"https://d.weibo.com/1087030002_2975_1003_0","cat":"明星","total":100}];
cats.forEach((cat)=>{
let len = cat.total;
for(var i=1;i<len+1;i++){
pool.submit(config,Object.assign({},cat,{url:`${cat.url}?page=${i}`}),callback)
}
})
|
