Skip to content

5
51business
Commit 92e33373 authored by 方斌's avatar 方斌
Browse files
Options

z

parent 617f35ac
Hide whitespace changes
Inline Side-by-side
Showing with 231 additions and 16 deletions
app/controller/house/tool.js
View file @ 92e33373
...@@ -93,8 +93,25 @@ class ToolController extends Controller { ...@@ -93,8 +93,25 @@ class ToolController extends Controller {
async calculateHousePrice2() { async calculateHousePrice2() {
const { ctx } = this; const { ctx } = this;
var Spider = require('node-spider'); var Spider = require('node-spider');
const data = {
userName: '',
gardenId: '',
gardenInternalID: 'c1283359-9995-4a52-ae76-ca06135b825c',
name: '中天九溪诚品',
area: 100,
floor: 2,
totlaFloor: 6,
directionStr: 'SOUTHEAST',
buildingId: '',
building: '',
roomNoId: '',
roomNo: '',
subQueryType: 'quick',
}
await this.httprequest('https://hangzhou.qfang.com/fangjia/chaxun?userName=&gardenId=&gardenInternalID=F3365A31BBB6F477E040A8C112002A44&name=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD&area=100&floor=1&totlaFloor=12&directionStr=NORTHSOUTH&buildingId=&building=&roomNoId=&roomNo=&subQueryType=quick', data);
return;
var spider = new Spider({ var spider = new Spider({
// How many requests can be run in parallel // How many requests can be run in parallel
concurrent: 5, concurrent: 5,
...@@ -120,7 +137,8 @@ class ToolController extends Controller { ...@@ -120,7 +137,8 @@ class ToolController extends Controller {
done: function () { done: function () {
}, },
method: 'post', method: 'POST',
body: data,
headers: { headers: {
'Host': 'hangzhou.qfang.com', 'Host': 'hangzhou.qfang.com',
'Connection': 'keep-alive', 'Connection': 'keep-alive',
...@@ -152,22 +170,218 @@ class ToolController extends Controller { ...@@ -152,22 +170,218 @@ class ToolController extends Controller {
encoding: 'utf8' encoding: 'utf8'
}); });
var handleRequest = function (doc) { // var handleRequest = function (doc) {
// new page crawled // // new page crawled
console.log(doc.res); // response object // console.log(doc.res); // response object
// console.log(doc.url); // page url // // console.log(doc.url); // page url
// uses cheerio, check its docs for more info // // uses cheerio, check its docs for more info
// doc.$('a').each(function (i, elem) { // // doc.$('a').each(function (i, elem) {
// // do stuff with element // // // do stuff with element
// var href = doc.$(elem).attr('href').split('#')[0]; // // var href = doc.$(elem).attr('href').split('#')[0];
// var url = doc.resolve(href); // // var url = doc.resolve(href);
// // crawl more // // // crawl more
// spider.queue(url, handleRequest); // // spider.queue(url, handleRequest);
// }); // // });
}; // };
// const data = {
// userName: '',
// gardenId: '',
// gardenInternalID: 'c1283359-9995-4a52-ae76-ca06135b825c',
// name: '中天九溪诚品',
// area: 100,
// floor: 2,
// totlaFloor: 6,
// directionStr: 'SOUTHEAST',
// buildingId: '',
// building: '',
// roomNoId: '',
// roomNo: '',
// subQueryType: 'quick',
// }
// const header = {
// 'Host': 'hangzhou.qfang.com',
// 'Connection': 'keep-alive',
// 'Content-Length': '242',
// 'Cache-Control': 'max-age=0',
// 'Origin': 'https://hangzhou.qfang.com',
// 'Upgrade-Insecure-Requests': 1,
// 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER',
// 'Content-Type': 'application/x-www-form-urlencoded',
// 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
// 'Referer': 'https://hangzhou.qfang.com/fangjia/area',
// 'Accept-Encoding': 'gzip, deflate, br',
// 'Accept-Language': 'zh-CN,zh;q=0.8',
// 'Cookie': 'acw_tc=0e1d391d15550609946915691ecd9bd6cfe0fae2fd1056fc4cc49c0a89; sid=72d0d94c-dd84-4fcb-a21c-29c0aa101544; cookieId=75f8366e-31ce-43db-bb64-a9f41df1ce4f; qchatid=92e2bc4e-a998-448a-8a5c-c54344a88c0c; language=SIMPLIFIED; JSESSIONID=aaaPSWGfw3iIxdW362qOw; looks=GARDEN%2C1758%2C1758%7CGARDEN%2C4552195%2C4552195; sec_tc=AQAAACYaNEoaMwgAeuCC4l0hf8QCTSkM; quickGardenHistory=%5B%7B%22name%22%3A%22%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81%22%2C%22id%22%3A%22c1283359-9995-4a52-ae76-ca06135b825c%22%7D%2C%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; HOUSE_PRICE_TOKEN=c1ea3ec587eb45aea1bfd9838c65e279; _ga=GA1.3.1611092705.1555060995; _gid=GA1.3.1809773729.1555314385; WINDOW_DEVICE_PIXEL_RATIO=1; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1555060996; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555394042',
// // 'host': 'hangzhou.qfang.com',
// // 'connection': 'keep-alive',
// // 'content-length': '242',
// // 'cache-control': 'max-age=0',
// // 'origin': 'https://hangzhou.qfang.com',
// // 'upgrade-insecure-requests': 1,
// // 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER',
// // 'content-type': 'application/x-www-form-urlencoded',
// // 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
// // 'referer': 'https://hangzhou.qfang.com/fangjia/area',
// // 'accept-encoding': 'gzip, deflate, br',
// // 'accept-language': 'zh-CN,zh;q=0.8',
// // 'cookie': 'acw_tc=0e1d391d15550609946915691ecd9bd6cfe0fae2fd1056fc4cc49c0a89; sid=72d0d94c-dd84-4fcb-a21c-29c0aa101544; cookieId=75f8366e-31ce-43db-bb64-a9f41df1ce4f; qchatid=92e2bc4e-a998-448a-8a5c-c54344a88c0c; language=SIMPLIFIED; JSESSIONID=aaaPSWGfw3iIxdW362qOw; looks=GARDEN%2C1758%2C1758%7CGARDEN%2C4552195%2C4552195; sec_tc=AQAAACYaNEoaMwgAeuCC4l0hf8QCTSkM; quickGardenHistory=%5B%7B%22name%22%3A%22%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81%22%2C%22id%22%3A%22c1283359-9995-4a52-ae76-ca06135b825c%22%7D%2C%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; HOUSE_PRICE_TOKEN=c1ea3ec587eb45aea1bfd9838c65e279; _ga=GA1.3.1611092705.1555060995; _gid=GA1.3.1809773729.1555314385; WINDOW_DEVICE_PIXEL_RATIO=1; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1555060996; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555394042',
// }
// let ret2 = await ctx.curl('https://hangzhou.qfang.com/fangjia/chaxun', { method: 'POST', data: data, headers: header });
// console.log(ret2)
// start crawling // start crawling
spider.queue('https://hangzhou.qfang.com/fangjia/chaxun?gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick', handleRequest);
//spider.queue('https://hangzhou.qfang.com/fangjia/chaxun', handleRequest);
}
async httprequest(url,data){
const { ctx } = this;
let ret2 = await ctx.curl('https://hangzhou.qfang.com/fangjia/area', { method: 'POST', data: data});
console.log(ret2.headers['set-cookie'])
var cookies = ret2.headers['set-cookie'];
var cookie = '';
for(let item in cookies) {
let cookie_items = cookies[item].split(';');
cookie += cookie_items[0] + '; ';
}
//var cookie = ret2.headers['set-cookie'].join(";");
// console.log(cookie);
//return
// var https = require('https');
// var querystring = require('querystring');
// // form data
// var postData = querystring.stringify(data);
// // request option
// var options = {
// host: 'hangzhou.qfang.com',
// port: 443,
// method: 'POST',
// path: '/fangjia/chaxun',
// headers: {
// 'Content-Type': 'application/x-www-form-urlencoded',
// 'Content-Length': postData.length,
// 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
// 'accept-encoding': 'gzip, deflate, br',
// 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
// 'cookie': 'acw_tc=755bc0cd15535143523154399e729103d77373973de8985880cd7a47bc;cookieId=67f995ee-e083-44fb-b182-db029e5437d6; qchatid=34e9f77d-e899-4b7e-a4fb-cbc51bf40e04; _ga=GA1.3.1512146760.1553514363; WINDOW_DEVICE_PIXEL_RATIO=1; sid=017b524d-b74b-4a3a-89db-c5c5e4eb261d; language=SIMPLIFIED; JSESSIONID=aaa50dKCR89Ip7UNxArOw; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1553514365,1555070292; quickGardenHistory=%5B%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; historyKeywords_HANGZHOU_SALE=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; ROOM_SALE=%2Fsale%3Fkeyword%3D%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%5E%20%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; _gid=GA1.3.862841217.1555398388; HOUSE_PRICE_TOKEN=ee850cf8ab844430bebbec7bd5e3bc7f; sec_tc=AQAAAJooC1zSAAMAeuCC4lb23vtydnhS; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555403781',
// 'upgrade-insecure-requests': 1,
// 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
// }
// };
// // request object
// var req = https.request(options, function (res) {
// var result = '';
// res.on('data', function (chunk) {
// result += chunk;
// });
// res.on('end', function () {
// console.log(result);
// });
// res.on('error', function (err) {
// console.log(err);
// })
// });
// // req error
// req.on('error', function (err) {
// console.log(err);
// });
// //send request witht the postData form
// req.write(postData);
// req.end();
var request = require('request');
//var j = request.jar()
//var cookie_string = await this.get_cookie();
// request({url: 'https://hangzhou.qfang.com/fangjia/area', jar: j}, function () {
// var cookie_string = j.getCookieString(url); // "key1=value1; key2=value2; ..."
// //var cookies = j.getCookies(url);
// //console.log(cookie_string)
// })
//console.log(cookie_string)
//var jar = await this.setJar(request.jar());
request({
url: url,
jar: request.jar(),
headers: {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
'cache-control': 'max-age=0',
'cookie': cookie,
'referer': 'https://hangzhou.qfang.com/fangjia/chaxun?userName=&gardenId=&gardenInternalID=F3365A31BBB6F477E040A8C112002A44&name=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD&area=100&floor=1&totlaFloor=12&directionStr=NORTHSOUTH&buildingId=&building=&roomNoId=&roomNo=&subQueryType=quick',
'upgrade-insecure-requests': 1,
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
}
// method: 'POST',
// //form: data,
// //rejectUnauthorized : false,
// json: true,
// headers: {
// 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
// 'accept-encoding': 'gzip, deflate, br',
// 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
// 'cache-control': 'max-age=0',
// 'content-type': 'application/x-www-form-urlencoded',
// 'cookie': 'acw_tc=755bc0cd15535143523154399e729103d77373973de8985880cd7a47bc; cookieId=67f995ee-e083-44fb-b182-db029e5437d6; qchatid=34e9f77d-e899-4b7e-a4fb-cbc51bf40e04; _ga=GA1.3.1512146760.1553514363; WINDOW_DEVICE_PIXEL_RATIO=1; sid=017b524d-b74b-4a3a-89db-c5c5e4eb261d; language=SIMPLIFIED; JSESSIONID=aaa50dKCR89Ip7UNxArOw; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1553514365,1555070292; quickGardenHistory=%5B%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; historyKeywords_HANGZHOU_SALE=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; ROOM_SALE=%2Fsale%3Fkeyword%3D%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%5E%20%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; _gid=GA1.3.862841217.1555398388; sec_tc=AQAAABeqDDL1SAcAeuCC4kBI21MsrAca; HOUSE_PRICE_TOKEN=e9be264a6cf5495d96c7cb3efcfa1268; _dc_gtm_UA-47416713-1=1; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555400841',
// 'origin': 'https://hangzhou.qfang.com',
// 'referer': 'https://hangzhou.qfang.com/fangjia/chaxun',
// 'upgrade-insecure-requests': 1,
// 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 ' ,
// },
// body: JSON.stringify(data),
}, function(error, response, body) {
//console.log(error)
console.log(response)
if (!error && response.statusCode == 200) {
var resultStr = body.replace(/\ +/g, ""); //去掉空格
resultStr = resultStr.replace(/[ ]/g, ""); //去掉空格
resultStr = resultStr.replace(/[\r\n]/g, ""); //去掉回车换行
resultStr = resultStr.replace(/[\n]/g, ""); //去掉换行
resultStr = resultStr.replace(/[\r]/g, ""); //去掉回车
//console.log(resultStr) // 请求成功的处理逻辑
}
});
};
async get_cookie() {
var request = require('request');
var j = request.jar()
//var cookie_string = '';
request({url: 'https://hangzhou.qfang.com/fangjia/area', jar: j}, function () {
var cookie_string = j.getCookieString(url); // "key1=value1; key2=value2; ..."
})
//request.on('data', )
}
async getJar() {
var request = require('request');
var jar;
if(jar)
return jar;
else {
jar = request.jar();
return jar;
}
}
async setJar(jarParam) {
jar = jarParam;
} }
async calculateHousePrice() { async calculateHousePrice() {
......
app/controller/official/jobtype.js
View file @ 92e33373
...@@ -4,6 +4,7 @@ const Controller = require('egg').Controller; ...@@ -4,6 +4,7 @@ const Controller = require('egg').Controller;
class JobtypeController extends Controller { class JobtypeController extends Controller {
async list() { async list() {
const { ctx, service } = this; const { ctx, service } = this;
if (ctx.params.jobtype === 'undefined') { return ctx.throw(422, 'job_style is missing!'); } if (ctx.params.jobtype === 'undefined') { return ctx.throw(422, 'job_style is missing!'); }
let job_type = ctx.params.jobtype; let job_type = ctx.params.jobtype;
const ret = await service.official.job.type.get_list( { job_type: job_type } ); const ret = await service.official.job.type.get_list( { job_type: job_type } );
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment