Skip to content

5
51business
Commit 617f35ac authored by 李尚科's avatar 李尚科
Browse files
Options

tool fix

parent 4e5fa132
Pipeline #6025 passed with stage
    in 10 minutes 23 seconds
    Show whitespace changes
    Inline Side-by-side
    Showing with 214 additions and 3 deletions
    app/controller/house/tool.js
    View file @ 617f35ac
    'use strict'; 'use strict';
    const Controller = require('egg').Controller; const Controller = require('egg').Controller;
    const http = require('http');
    const process = require('child_process');
    const Spider = require('node-spider');
    class ToolController extends Controller { class ToolController extends Controller {
    //获取地图上的点位 //获取地图上的点位
    ...@@ -79,7 +81,7 @@ class ToolController extends Controller { ...@@ -79,7 +81,7 @@ class ToolController extends Controller {
    ret = await ctx.blockModel.HousePriceTrend.one({ where: { id: area_code } }); ret = await ctx.blockModel.HousePriceTrend.one({ where: { id: area_code } });
    ret = ret.trend_json; ret = ret.trend_json;
    } else if (type === 'increase') { } else if (type === 'increase') {
    ret = await ctx.blockModel.HousePriceIncrease.one({ where: { id: area_code } }); ret = await ctx.blockModel.HouseQuestion.one({ where: { id: area_code } });
    ret = ret.increase_json; ret = ret.increase_json;
    } else if (type === 'question') { } else if (type === 'question') {
    ret = await ctx.blockModel.HouseQuestion.one({ where: { id: area_code } }); ret = await ctx.blockModel.HouseQuestion.one({ where: { id: area_code } });
    ...@@ -88,6 +90,192 @@ class ToolController extends Controller { ...@@ -88,6 +90,192 @@ class ToolController extends Controller {
    ctx.success(ret); ctx.success(ret);
    } }
    async calculateHousePrice2() {
    const { ctx } = this;
    var Spider = require('node-spider');
    var spider = new Spider({
    // How many requests can be run in parallel
    concurrent: 5,
    // How long to wait after each request
    delay: 0,
    // A stream to where internal logs are sent, optional
    logs: process.stderr,
    // Re-visit visited URLs, false by default
    allowDuplicates: false,
    // If `true` all queued handlers will be try-catch'd, errors go to `error` callback
    catchErrors: true,
    // If `true` the spider will set the Referer header automatically on subsequent requests
    addReferrer: true,
    // If `true` adds the X-Requested-With:XMLHttpRequest header
    xhr: false,
    // If `true` adds the Connection:keep-alive header and forever option on request module
    keepAlive: false,
    // Called when there's an error, throw will be used if none is provided
    error: function (err, url) {
    },
    // Called when there are no more requests
    done: function () {
    },
    method: 'post',
    headers: {
    'Host': 'hangzhou.qfang.com',
    'Connection': 'keep-alive',
    'Content-Length': '242',
    'Cache-Control': 'max-age=0',
    'Origin': 'https://hangzhou.qfang.com',
    'Upgrade-Insecure-Requests': 1,
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER',
    'Content-Type': 'application/x-www-form-urlencoded',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Referer': 'https://hangzhou.qfang.com/fangjia/area',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.8',
    'Cookie': 'acw_tc=0e1d391d15550609946915691ecd9bd6cfe0fae2fd1056fc4cc49c0a89; sid=72d0d94c-dd84-4fcb-a21c-29c0aa101544; cookieId=75f8366e-31ce-43db-bb64-a9f41df1ce4f; qchatid=92e2bc4e-a998-448a-8a5c-c54344a88c0c; language=SIMPLIFIED; JSESSIONID=aaaPSWGfw3iIxdW362qOw; looks=GARDEN%2C1758%2C1758%7CGARDEN%2C4552195%2C4552195; sec_tc=AQAAACYaNEoaMwgAeuCC4l0hf8QCTSkM; quickGardenHistory=%5B%7B%22name%22%3A%22%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81%22%2C%22id%22%3A%22c1283359-9995-4a52-ae76-ca06135b825c%22%7D%2C%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; HOUSE_PRICE_TOKEN=c1ea3ec587eb45aea1bfd9838c65e279; _ga=GA1.3.1611092705.1555060995; _gid=GA1.3.1809773729.1555314385; WINDOW_DEVICE_PIXEL_RATIO=1; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1555060996; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555394042',
    // 'host': 'hangzhou.qfang.com',
    // 'connection': 'keep-alive',
    // 'content-length': '242',
    // 'cache-control': 'max-age=0',
    // 'origin': 'https://hangzhou.qfang.com',
    // 'upgrade-insecure-requests': 1,
    // 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER',
    // 'content-type': 'application/x-www-form-urlencoded',
    // 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    // 'referer': 'https://hangzhou.qfang.com/fangjia/area',
    // 'accept-encoding': 'gzip, deflate, br',
    // 'accept-language': 'zh-CN,zh;q=0.8',
    // 'cookie': 'acw_tc=0e1d391d15550609946915691ecd9bd6cfe0fae2fd1056fc4cc49c0a89; sid=72d0d94c-dd84-4fcb-a21c-29c0aa101544; cookieId=75f8366e-31ce-43db-bb64-a9f41df1ce4f; qchatid=92e2bc4e-a998-448a-8a5c-c54344a88c0c; language=SIMPLIFIED; JSESSIONID=aaaPSWGfw3iIxdW362qOw; looks=GARDEN%2C1758%2C1758%7CGARDEN%2C4552195%2C4552195; sec_tc=AQAAACYaNEoaMwgAeuCC4l0hf8QCTSkM; quickGardenHistory=%5B%7B%22name%22%3A%22%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81%22%2C%22id%22%3A%22c1283359-9995-4a52-ae76-ca06135b825c%22%7D%2C%7B%22name%22%3A%22%E5%9F%8E%E4%B8%AD%E8%8A%B1%E5%9B%AD%22%2C%22id%22%3A%22F3365A31BBB6F477E040A8C112002A44%22%7D%5D; HOUSE_PRICE_TOKEN=c1ea3ec587eb45aea1bfd9838c65e279; _ga=GA1.3.1611092705.1555060995; _gid=GA1.3.1809773729.1555314385; WINDOW_DEVICE_PIXEL_RATIO=1; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1555060996; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1555394042',
    },
    encoding: 'utf8'
    });
    var handleRequest = function (doc) {
    // new page crawled
    console.log(doc.res); // response object
    // console.log(doc.url); // page url
    // uses cheerio, check its docs for more info
    // doc.$('a').each(function (i, elem) {
    // // do stuff with element
    // var href = doc.$(elem).attr('href').split('#')[0];
    // var url = doc.resolve(href);
    // // crawl more
    // spider.queue(url, handleRequest);
    // });
    };
    // start crawling
    spider.queue('https://hangzhou.qfang.com/fangjia/chaxun?gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick', handleRequest);
    }
    async calculateHousePrice() {
    const { ctx } = this;
    const input_params = ctx.request.body;
    const rule = {
    area: { type: 'string', required: true },
    direction_str: { type: 'string', required: true },
    floor: { type: 'string', required: true },
    total_floor: { type: 'string', required: true },
    };
    // ctx.validate(rule, input_params);
    // const qfang_url = 'https://hangzhou.qfang.com/fangjia/area';
    const qfang_url2 = 'https://hangzhou.qfang.com/fangjia/chaxun?gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick';
    // const qfang_url2 = 'https://hangzhou.qfang.com/fangjia/chaxun?gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick';
    // const qfang_url2 = 'https://hangzhou.qfang.com/fangjia/chaxun?gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=%E4%B8%AD%E5%A4%A9%E4%B9%9D%E6%BA%AA%E8%AF%9A%E5%93%81&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick';
    const data = {
    // userName: '',
    // gardenId: '',
    gardenInternalID: 'c1283359-9995-4a52-ae76-ca06135b825c',
    name: '中天九溪诚品',
    area: input_params.area || 100,
    floor: input_params.floor || 2,
    totlaFloor: input_params.total_floor || 6,
    directionStr: input_params.direction_str || 'SOUTHEAST',
    // buildingId: '',
    // building: '',
    // roomNoId: '',
    // roomNo: '',
    subQueryType: 'quick',
    }
    // await process.exec('curl -c cookie.txt -L -d "gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick" https://hangzhou.qfang.com/fangjia/chaxun', function (error, stdout, stderr) {
    // console.info(error);
    // console.info(stderr);
    // if (error === null) {
    // // process.exec('curl -b cookie.txt -L -d "gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick" https://hangzhou.qfang.com/fangjia/chaxun >> test.html', function (error2, stdout2, stderr2) {
    // // console.info(11111);
    // // });
    // }
    // });
    // await process.exec('curl -b cookie.txt -L -d "gardenInternalID=c1283359-9995-4a52-ae76-ca06135b825c&name=中天九溪诚品&area=100&floor=2&totlaFloor=6&directionStr=SOUTHEAST&subQueryType=quick" https://hangzhou.qfang.com/fangjia/chaxun >> test.html', function(error, stdout, stderr){
    // console.info(111111);
    // });
    // console.info(22222);
    // ctx.success(12);
    // return;
    // const request = (url) => http.get(qfang_url, function (resp) {
    // let chunks = [];
    // let size = 0;
    // console.info('11111111111111111111111111111');
    // console.info(resp.headers);
    // if(resp.statusCode == 301 || resp.statusCode == 302){
    // const redirectUrl = resp.headers['location'];
    // request(redirectUrl);
    // } else {
    // resp.on('data', function (chunk) {
    // chunks.push(chunk);
    // size += chunk.length;
    // });
    // resp.on('end', function () {
    // let data = Buffer.concat(chunks, size);
    // let html = data.toString();
    // console.info(html);
    // });
    // }
    // });
    // request(qfang_url);
    // console.info(2222);
    // ctx.success(11);
    // return;
    let ret1 = await ctx.curl(qfang_url2, { method: 'POST', data: data });
    let ret2 = await ctx.curl(ret1.headers.location, { method: 'POST', data: data });
    // const headers = ret1.headers;
    // const cookies = fs.readFileSync('cookie.txt').toString();
    // let ret = await ctx.curl(qfang_url2, { method: 'POST', data: data, headers: { Cookie: cookies } });
    // const html = ret.data.toString('utf8');
    // ctx.success(html);
    // return;
    ctx.success({ ret2 });
    }
    async getQFangAreaList() {
    const { ctx } = this;
    const input_parmas = ctx.query;
    const rule = {
    keyword: { type: 'string', required: true },
    }
    ctx.validate(rule, input_parmas);
    const qfang_url = 'https://hangzhou.qfang.com/fangjia/getGarden';
    const keyword = encodeURIComponent(input_parmas.keyword);
    const result = await ctx.helper.send_request(qfang_url + '?keyword=' + keyword, {}, { dataType: 'json' });
    let ret = [];
    const headers = result.headers;
    if (result.status === 200 && result.data && result.data.items) {
    const area_items = result.data.items;
    for (let i in area_items) {
    const area_item = area_items[i];
    }
    }
    ctx.success({ headers, ret: result.data });
    }
    } }
    module.exports = ToolController; module.exports = ToolController;
    app/router/house.js
    View file @ 617f35ac
    ...@@ -6,7 +6,9 @@ module.exports = app => { ...@@ -6,7 +6,9 @@ module.exports = app => {
    router.get('/tool/:house_style/:area_code/:level', 'house.tool.getMapPoint');//购房计划地图点位 router.get('/tool/:house_style/:area_code/:level', 'house.tool.getMapPoint');//购房计划地图点位
    router.post('/tool/plan', 'house.tool.generateBuyHousePlan');//生成购房计划 router.post('/tool/plan', 'house.tool.generateBuyHousePlan');//生成购房计划
    router.get('/tool/:type/:area_code', 'house.tool.getHousePriceFeature');//房价指数 涨跌 router.get('/tool/:type/:area_code', 'house.tool.getHousePriceFeature');//房价指数 房价涨跌 购房资格、贷款额度问答
    router.get('/tool/calculate_price', 'house.tool.calculateHousePrice2');//房产估价
    router.get('/tool/qfang_area_list', 'house.tool.getQFangAreaList');//房产估价
    //租房 //租房
    router.get('/rental_house/home', 'house.rentalHouse.home');//租房首页信息 router.get('/rental_house/home', 'house.rentalHouse.home');//租房首页信息
    ......
    app/service/house/rental_house.js
    View file @ 617f35ac
    ...@@ -53,6 +53,14 @@ class RentalHouseService extends Service { ...@@ -53,6 +53,14 @@ class RentalHouseService extends Service {
    operator: 'equal', operator: 'equal',
    }); });
    } }
    if (condition.name) {
    await service.house.searchHistory.addSearchHistory({ type: 2, key_word: condition.name });//增加搜索历史
    queryConditions.push({
    key: 'name',
    value: condition.name,
    operator: 'contains',
    });
    }
    queryConditions.push({ queryConditions.push({
    key: "state", key: "state",
    value: 1, value: 1,
    ......
    cookie.txt 0 → 100644
    View file @ 617f35ac
    # Netscape HTTP Cookie File
    # http://curl.haxx.se/docs/http-cookies.html
    # This file was generated by libcurl! Edit at your own risk.
    #HttpOnly_hangzhou.qfang.com FALSE / FALSE 1555337519 sec_tc AQAAAE+G4wNvjgoAeuCC4ndGDrZ0m3RX
    #HttpOnly_hangzhou.qfang.com FALSE / FALSE 1558015320 acw_tc 7250b31815553369196918090e18c441c8b7d4f052c9fce70140ddae7e
    #HttpOnly_.qfang.com TRUE / TRUE 1555941719 sid 51f92bcc-83fd-46e4-ba7f-85a4d4f00d8a
    #HttpOnly_.qfang.com TRUE / TRUE 1870696919 cookieId 2be15c40-035e-4c13-b9d8-56ad9228967b
    .qfang.com TRUE / TRUE 1870696919 qchatid d2d2a2fc-9c79-42ae-a293-3694c2398818
    .qfang.com TRUE / TRUE 1556546519 language SIMPLIFIED
    hangzhou.qfang.com FALSE / FALSE 0 JSESSIONID aaa1w9XtM3clW9vSFtHOw
    #HttpOnly_.qfang.com TRUE / TRUE 1555423320 HOUSE_PRICE_TOKEN d90dae9b363f479a9895e2f5a89256ba
    package.json
    View file @ 617f35ac
    ...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
    "moment": "^2.22.2", "moment": "^2.22.2",
    "mysql2": "^1.6.1", "mysql2": "^1.6.1",
    "node-cache": "^4.2.0", "node-cache": "^4.2.0",
    "node-spider": "^1.4.1",
    "parameter": "^3.3.1", "parameter": "^3.3.1",
    "ramda": "^0.25.0", "ramda": "^0.25.0",
    "uuid": "^3.3.2" "uuid": "^3.3.2"
    ......
    test.html 0 → 100644
    View file @ 617f35ac
    This source diff could not be displayed because it is too large. You can view the blob instead.
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment