Skip to content

5
51business
Commit 75b553c1 authored by Hsinli's avatar Hsinli
Browse files
Options

Merge branch 'master' of t-git.51gjj.com:fangbin/51business

parents 8d7903f4 65932fa9
Pipeline #6052 passed with stage
    in 8 seconds
    Show whitespace changes
    Inline Side-by-side
    Showing with 230 additions and 15 deletions
    app/controller/house/tool.js
    View file @ 75b553c1
    ......@@ -93,8 +93,25 @@ class ToolController extends Controller {
    async calculateHousePrice2() {
    const { ctx } = this;
    var Spider = require('node-spider');
    var Spider = require('node-spider');
    const data = {
    userName: '',
    gardenId: '',
    gardenInternalID: 'c1283359-9995-4a52-ae76-ca06135b825c',
    name: '中天九溪诚品',
    area: 100,
    floor: 2,
    totlaFloor: 6,
    directionStr: 'SOUTHEAST',
    buildingId: '',
    building: '',
    roomNoId: '',
    roomNo: '',
    subQueryType: 'quick',
    }
    await this.httprequest('https://hangzhou.qfang.com/fangjia/chaxun?userName=&gardenId=&gardenInternalID=F3365A31BBB6F477E040A8C112002A44&name=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD&area=100&floor=1&totlaFloor=12&directionStr=NORTHSOUTH&buildingId=&building=&roomNoId=&roomNo=&subQueryType=quick', data);
    return;
    var spider = new Spider({
    // How many requests can be run in parallel
    concurrent: 5,
    ......@@ -120,7 +137,8 @@ class ToolController extends Controller {
    done: function () {
    },
    method: 'post',
    method: 'POST',
    body: data,
    headers: {
    'Host': 'hangzhou.qfang.com',
    'Connection': 'keep-alive',
    ......@@ -152,22 +170,218 @@ class ToolController extends Controller {
    encoding: 'utf8'
    });
    var handleRequest = function (doc) {
    // new page crawled
    console.log(doc.res); // response object
    // console.log(doc.url); // page url
    // uses cheerio, check its docs for more info
    // doc.$('a').each(function (i, elem) {
    // // do stuff with element
    // var href = doc.$(elem).attr('href').split('#')[0];
    // var url = doc.resolve(href);
    // // crawl more
    // spider.queue(url, handleRequest);
    // var handleRequest = function (doc) {
    // // new page crawled
    // console.log(doc.res); // response object
    // // console.log(doc.url); // page url
    // // uses cheerio, check its docs for more info
    // // doc.$('a').each(function (i, elem) {
    // // // do stuff with element
    // // var href = doc.$(elem).attr('href').split('#')[0];
    // // var url = doc.resolve(href);
    // // // crawl more
    // // spider.queue(url, handleRequest);
    // // });
    // };
    // const data = {
    // userName: '',
    // gardenId: '',
    // gardenInternalID: 'c1283359-9995-4a52-ae76-ca06135b825c',
    // name: '中天九溪诚品',
    // area: 100,
    // floor: 2,
    // totlaFloor: 6,
    // directionStr: 'SOUTHEAST',
    // buildingId: '',
    // building: '',
    // roomNoId: '',
    // roomNo: '',
    // subQueryType: 'quick',
    // }
    // const header = {
    // 'Host': 'hangzhou.qfang.com',
    // 'Connection': 'keep-alive',
    // 'Content-Length': '242',
    // 'Cache-Control': 'max-age=0',
    // 'Origin': 'https://hangzhou.qfang.com',
    // 'Upgrade-Insecure-Requests': 1,
    // 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER',
    // 'Content-Type': 'application/x-www-form-urlencoded',
    // 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    // 'Referer': 'https://hangzhou.qfang.com/fangjia/area',
    // 'Accept-Encoding': 'gzip, deflate, br',
    // 'Accept-Language': 'zh-CN,zh;q=0.8',
    // 'Cookie': 'acw_tc=0e1d391d15550609946915691ecd9bd6cfe0fae2fd1056fc4cc49c0a89; sid=72d0d94c-dd84-4fcb-a21c-29c0aa101544; cookieId=75f8366e-31ce-43db-bb64-a9f41df1ce4f; qchatid=92e2bc4e-a998-448a-8a5c-c54344a88c0c; language=SIMPLIFIED; JSESSIONID=aaaPSWGfw3iIxdW362qOw; looks=GARDEN%2C1758%2C1758%7CGARDEN%2C4552195%2C4552195; sec_tc=AQAAACYaNEoaMwgAeuCC4l0hf8QCTSkM; quickGardenHistory=%5B%7B%22name%22%3A%22%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81%22%2C%22id%22%3A%22c1283359-9995-4a52-ae76-ca06135b825c%22%7D%2C%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; HOUSE_PRICE_TOKEN=c1ea3ec587eb45aea1bfd9838c65e279; _ga=GA1.3.1611092705.1555060995; _gid=GA1.3.1809773729.1555314385; WINDOW_DEVICE_PIXEL_RATIO=1; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1555060996; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555394042',
    // // 'host': 'hangzhou.qfang.com',
    // // 'connection': 'keep-alive',
    // // 'content-length': '242',
    // // 'cache-control': 'max-age=0',
    // // 'origin': 'https://hangzhou.qfang.com',
    // // 'upgrade-insecure-requests': 1,
    // // 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER',
    // // 'content-type': 'application/x-www-form-urlencoded',
    // // 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    // // 'referer': 'https://hangzhou.qfang.com/fangjia/area',
    // // 'accept-encoding': 'gzip, deflate, br',
    // // 'accept-language': 'zh-CN,zh;q=0.8',
    // // 'cookie': 'acw_tc=0e1d391d15550609946915691ecd9bd6cfe0fae2fd1056fc4cc49c0a89; sid=72d0d94c-dd84-4fcb-a21c-29c0aa101544; cookieId=75f8366e-31ce-43db-bb64-a9f41df1ce4f; qchatid=92e2bc4e-a998-448a-8a5c-c54344a88c0c; language=SIMPLIFIED; JSESSIONID=aaaPSWGfw3iIxdW362qOw; looks=GARDEN%2C1758%2C1758%7CGARDEN%2C4552195%2C4552195; sec_tc=AQAAACYaNEoaMwgAeuCC4l0hf8QCTSkM; quickGardenHistory=%5B%7B%22name%22%3A%22%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81%22%2C%22id%22%3A%22c1283359-9995-4a52-ae76-ca06135b825c%22%7D%2C%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; HOUSE_PRICE_TOKEN=c1ea3ec587eb45aea1bfd9838c65e279; _ga=GA1.3.1611092705.1555060995; _gid=GA1.3.1809773729.1555314385; WINDOW_DEVICE_PIXEL_RATIO=1; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1555060996; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555394042',
    // }
    // let ret2 = await ctx.curl('https://hangzhou.qfang.com/fangjia/chaxun', { method: 'POST', data: data, headers: header });
    // console.log(ret2)
    // start crawling
    //spider.queue('https://hangzhou.qfang.com/fangjia/chaxun', handleRequest);
    }
    async httprequest(url,data){
    const { ctx } = this;
    let ret2 = await ctx.curl('https://hangzhou.qfang.com/fangjia/area', { method: 'POST', data: data});
    console.log(ret2.headers['set-cookie'])
    var cookies = ret2.headers['set-cookie'];
    var cookie = '';
    for(let item in cookies) {
    let cookie_items = cookies[item].split(';');
    cookie += cookie_items[0] + '; ';
    }
    //var cookie = ret2.headers['set-cookie'].join(";");
    // console.log(cookie);
    //return
    // var https = require('https');
    // var querystring = require('querystring');
    // // form data
    // var postData = querystring.stringify(data);
    // // request option
    // var options = {
    // host: 'hangzhou.qfang.com',
    // port: 443,
    // method: 'POST',
    // path: '/fangjia/chaxun',
    // headers: {
    // 'Content-Type': 'application/x-www-form-urlencoded',
    // 'Content-Length': postData.length,
    // 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    // 'accept-encoding': 'gzip, deflate, br',
    // 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
    // 'cookie': 'acw_tc=755bc0cd15535143523154399e729103d77373973de8985880cd7a47bc;cookieId=67f995ee-e083-44fb-b182-db029e5437d6; qchatid=34e9f77d-e899-4b7e-a4fb-cbc51bf40e04; _ga=GA1.3.1512146760.1553514363; WINDOW_DEVICE_PIXEL_RATIO=1; sid=017b524d-b74b-4a3a-89db-c5c5e4eb261d; language=SIMPLIFIED; JSESSIONID=aaa50dKCR89Ip7UNxArOw; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1553514365,1555070292; quickGardenHistory=%5B%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; historyKeywords_HANGZHOU_SALE=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; ROOM_SALE=%2Fsale%3Fkeyword%3D%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%5E%20%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; _gid=GA1.3.862841217.1555398388; HOUSE_PRICE_TOKEN=ee850cf8ab844430bebbec7bd5e3bc7f; sec_tc=AQAAAJooC1zSAAMAeuCC4lb23vtydnhS; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555403781',
    // 'upgrade-insecure-requests': 1,
    // 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    // }
    // };
    // // request object
    // var req = https.request(options, function (res) {
    // var result = '';
    // res.on('data', function (chunk) {
    // result += chunk;
    // });
    // res.on('end', function () {
    // console.log(result);
    // });
    // res.on('error', function (err) {
    // console.log(err);
    // })
    // });
    // // req error
    // req.on('error', function (err) {
    // console.log(err);
    // });
    // //send request witht the postData form
    // req.write(postData);
    // req.end();
    var request = require('request');
    //var j = request.jar()
    //var cookie_string = await this.get_cookie();
    // request({url: 'https://hangzhou.qfang.com/fangjia/area', jar: j}, function () {
    // var cookie_string = j.getCookieString(url); // "key1=value1; key2=value2; ..."
    // //var cookies = j.getCookies(url);
    // //console.log(cookie_string)
    // })
    //console.log(cookie_string)
    //var jar = await this.setJar(request.jar());
    request({
    url: url,
    jar: request.jar(),
    headers: {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'cache-control': 'max-age=0',
    'cookie': cookie,
    'referer': 'https://hangzhou.qfang.com/fangjia/chaxun?userName=&gardenId=&gardenInternalID=F3365A31BBB6F477E040A8C112002A44&name=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD&area=100&floor=1&totlaFloor=12&directionStr=NORTHSOUTH&buildingId=&building=&roomNoId=&roomNo=&subQueryType=quick',
    'upgrade-insecure-requests': 1,
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    }
    // method: 'POST',
    // //form: data,
    // //rejectUnauthorized : false,
    // json: true,
    // headers: {
    // 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    // 'accept-encoding': 'gzip, deflate, br',
    // 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
    // 'cache-control': 'max-age=0',
    // 'content-type': 'application/x-www-form-urlencoded',
    // 'cookie': 'acw_tc=755bc0cd15535143523154399e729103d77373973de8985880cd7a47bc; cookieId=67f995ee-e083-44fb-b182-db029e5437d6; qchatid=34e9f77d-e899-4b7e-a4fb-cbc51bf40e04; _ga=GA1.3.1512146760.1553514363; WINDOW_DEVICE_PIXEL_RATIO=1; sid=017b524d-b74b-4a3a-89db-c5c5e4eb261d; language=SIMPLIFIED; JSESSIONID=aaa50dKCR89Ip7UNxArOw; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1553514365,1555070292; quickGardenHistory=%5B%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; historyKeywords_HANGZHOU_SALE=%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; ROOM_SALE=%2Fsale%3Fkeyword%3D%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%5E%20%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD; _gid=GA1.3.862841217.1555398388; sec_tc=AQAAABeqDDL1SAcAeuCC4kBI21MsrAca; HOUSE_PRICE_TOKEN=e9be264a6cf5495d96c7cb3efcfa1268; _dc_gtm_UA-47416713-1=1; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555400841',
    // 'origin': 'https://hangzhou.qfang.com',
    // 'referer': 'https://hangzhou.qfang.com/fangjia/chaxun',
    // 'upgrade-insecure-requests': 1,
    // 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 ' ,
    // },
    // body: JSON.stringify(data),
    }, function(error, response, body) {
    //console.log(error)
    console.log(response)
    if (!error && response.statusCode == 200) {
    var resultStr = body.replace(/\ +/g, ""); //去掉空格
    resultStr = resultStr.replace(/[ ]/g, ""); //去掉空格
    resultStr = resultStr.replace(/[\r\n]/g, ""); //去掉回车换行
    resultStr = resultStr.replace(/[\n]/g, ""); //去掉换行
    resultStr = resultStr.replace(/[\r]/g, ""); //去掉回车
    //console.log(resultStr) // 请求成功的处理逻辑
    }
    });
    };
    // start crawling
    spider.queue('https://hangzhou.qfang.com/fangjia/chaxun?gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick', handleRequest);
    async get_cookie() {
    var request = require('request');
    var j = request.jar()
    //var cookie_string = '';
    request({url: 'https://hangzhou.qfang.com/fangjia/area', jar: j}, function () {
    var cookie_string = j.getCookieString(url); // "key1=value1; key2=value2; ..."
    })
    //request.on('data', )
    }
    async getJar() {
    var request = require('request');
    var jar;
    if(jar)
    return jar;
    else {
    jar = request.jar();
    return jar;
    }
    }
    async setJar(jarParam) {
    jar = jarParam;
    }
    async calculateHousePrice() {
    ......
    app/controller/official/jobtype.js
    View file @ 75b553c1
    ......@@ -4,6 +4,7 @@ const Controller = require('egg').Controller;
    class JobtypeController extends Controller {
    async list() {
    const { ctx, service } = this;
    if (ctx.params.jobtype === 'undefined') { return ctx.throw(422, 'job_style is missing!'); }
    let job_type = ctx.params.jobtype;
    const ret = await service.official.job.type.get_list( { job_type: job_type } );
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment