From 478a2b041c843161eacb6110193d08137e338c42 Mon Sep 17 00:00:00 2001 From: RainSun Date: Mon, 16 Nov 2020 16:05:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=99=BB=E5=BD=95=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/allFunction.py | 1 - lib/cache.py | 32 ++++++++++++++++++++++++++++++++ lib/crawler.py | 36 ++++++++++++++++-------------------- lib/crawler_test.py | 20 ++++++++++---------- lib/db.py | 8 +++----- 5 files changed, 61 insertions(+), 36 deletions(-) create mode 100644 lib/cache.py diff --git a/lib/allFunction.py b/lib/allFunction.py index 5fbd8c2..c23b21c 100644 --- a/lib/allFunction.py +++ b/lib/allFunction.py @@ -56,7 +56,6 @@ def manageScheduleGet(request): else: return {'errcode': 400, 'errmsg': '数据校验失败'} - # 工具函数 # MD5 校验 diff --git a/lib/cache.py b/lib/cache.py new file mode 100644 index 0000000..6dc8cdf --- /dev/null +++ b/lib/cache.py @@ -0,0 +1,32 @@ +import json +import requests +from urllib.parse import quote +import base64 +from bs4 import BeautifulSoup +import random +import sys + +# JSON转base64 +def btoa(content): + return base64.b64encode(quote(content).encode()) + +s = requests.Session() + +params = { + 'Dto': { + 'Code': "", + 'LoginName': "2017002372", + '​​PassWord': "102710", + } +} +params = str(btoa(json.dumps(params)))[2:-1] + +url = 'https://jwgl.cust.edu.cn/api/LoginApi/LocalLogin' + +params = { + "__log":{"Logtype":1}, + "param": params +} + +r = s.post(url=url, data=params) +print(r.text) \ No newline at end of file diff --git a/lib/crawler.py b/lib/crawler.py index 72f7cd3..d344fba 100644 --- a/lib/crawler.py +++ b/lib/crawler.py @@ -25,7 +25,7 @@ class Crawler(object): self.__session = requests.Session() # 获取统一身份系统的网页 r = self.__session.get( - url='http://mysso-cust-edu-cn-s.webvpn.cust.edu.cn:8118/cas/login?service=https%3A%2F%2Fwebvpn.cust.edu.cn%2Fauth%2Fcas_validate%3Fentry_id%3D1') + url='https://mysso.cust.edu.cn/cas/login?service=https://jwgls1.cust.edu.cn/welcome') soup = BeautifulSoup(r.text, 'html.parser') execution = soup.find_all(name='input')[6]['value'] formdata = { @@ -36,7 +36,7 @@ class Crawler(object): 'geolocation': '' } r = self.__session.post( - url='http://mysso-cust-edu-cn-s.webvpn.cust.edu.cn:8118/cas/login?service=https%3A%2F%2Fwebvpn.cust.edu.cn%2Fauth%2Fcas_validate%3Fentry_id%3D1', data=formdata) + url='https://mysso.cust.edu.cn/cas/login?service=https://jwgls1.cust.edu.cn/welcome', data=formdata) soup = BeautifulSoup(r.text, 'html.parser') flag = soup.find(name='title') if(flag.text == "手机号设置"): @@ -50,25 +50,24 @@ class Crawler(object): 'submit': '提交' } r = self.__session.post( - url="http://mysso-cust-edu-cn-s.webvpn.cust.edu.cn:8118/cas/login?service=https%3A%2F%2Fwebvpn.cust.edu.cn%2Fauth%2Fcas_validate%3Fentry_id%3D1", data=formdata) + url="https://mysso.cust.edu.cn/cas/login?service=https://jwgls1.cust.edu.cn/welcome", data=formdata) r = self.__session.get( - url='http://portal-cust-edu-cn-s.webvpn.cust.edu.cn:8118/custp/index') + url='https://portal.cust.edu.cn/custp/index') soup = BeautifulSoup(r.text, 'html.parser') try: - self.__ip = soup.findAll(name='a')[7]['href'][7:].split("-") + if soup.findAll(name='a')[4]['href'] != 'logout': + raise('账号或密码错误') except: return ('账号或者密码错误', 510) - r = self.__session.get(url='http://mysso-cust-edu-cn-s.webvpn.cust.edu.cn:8118/cas/login?service=http://' + - self.__ip[0] + '.' + self.__ip[1] + '.' + self.__ip[2] + '.' + self.__ip[3] + ':8080/welcome', allow_redirects=False) - ticket = r.headers['Location'][72:] - asp_net_sessionid_param = {'Ticket': ticket, 'Url': 'http://' + - self.__ip[0] + '.' + self.__ip[1] + '.' + self.__ip[2] + '.' + self.__ip[3] + ':8080/welcome'} + self.__ip = ['jwgls0', 'cust', 'edu', 'cn'] + r = self.__session.get(url='https://mysso.cust.edu.cn/cas/login?service=https://jwgls1.cust.edu.cn/welcome', allow_redirects=False) + ticket = r.headers['Location'][42:] + asp_net_sessionid_param = {'Ticket': ticket, 'Url': 'https://jwgls1.cust.edu.cn/welcome'} asp_net_sessionid_param = base64.b64encode( quote(json.dumps(asp_net_sessionid_param)).encode('utf-8')).decode('utf-8') asp_net_sessionid_param = {'param': asp_net_sessionid_param} headers = {'Content-Type': 'application/json'} - r = self.__session.post(url='http://' + self.__ip[0] + '-' + self.__ip[1] + '-' + self.__ip[2] + '-' + self.__ip[3] + - '-8080-p.webvpn.cust.edu.cn:8118/api/LoginApi/LGSSOLocalLogin?sf_request_type=ajax', data=json.dumps(asp_net_sessionid_param), headers=headers) + r = self.__session.post(url='https://jwgls1.cust.edu.cn/api/LoginApi/LGSSOLocalLogin?sf_request_type=ajax', data=json.dumps(asp_net_sessionid_param), headers=headers) data = json.loads(r.content.decode('utf-8')) # 提示未建立教务信息 if data['state'] == 1: @@ -80,12 +79,11 @@ class Crawler(object): print(e) return ('教务挂了', 512) - # 获取课表 ----------------------------------------------------------------------------- + # 获取成绩 ----------------------------------------------------------------------------- def getGrade(self): headers = {'Content-Type': 'application/json'} r = self.__session.post( - url='http://' + self.__ip[0] + '-' + self.__ip[1] + '-' + self.__ip[2] + '-' + self.__ip[3] + - '-8080-p.webvpn.cust.edu.cn:8118/api/ClientStudent/QueryService/GradeQueryApi/GetDataByStudent?sf_request_type=ajax', + url='https://jwgls1.cust.edu.cn/api/ClientStudent/QueryService/GradeQueryApi/GetDataByStudent?sf_request_type=ajax', data=json.dumps({"param": "JTdCJTIyU2hvd0dyYWRlVHlwZSUyMiUzQTElN0Q=", "__permission": {"MenuID": "4443798E-EB6E-4D88-BFBD-BB0A76FF6BD5", "Operation": 0}, "__log": {"MenuID": "4443798E-EB6E-4D88-BFBD-BB0A76FF6BD5", "Logtype": 6, "Context": "查询"}}), headers=headers @@ -233,20 +231,18 @@ class Crawler(object): } return ('ok', 200) - # 获取成绩 ----------------------------------------------------------------------------- + # 获取课表 ----------------------------------------------------------------------------- def getSchedule(self): headers = {'Content-Type': 'application/json'} r = self.__session.post( - url='http://'+self.__ip[0]+'-'+self.__ip[1]+'-'+self.__ip[2]+'-'+self.__ip[3] + - '-8080-p.webvpn.cust.edu.cn:8118/api/ClientStudent/Home/StudentHomeApi/GetHomeCurWeekTime?sf_request_type=ajax', + url='https://jwgls1.cust.edu.cn/api/ClientStudent/Home/StudentHomeApi/GetHomeCurWeekTime?sf_request_type=ajax', data=json.dumps({"param": "JTdCJTdE", "__permission": {"MenuID": "F71C97D5-D3E2-4FDA-9209-D7FA8626390E", "Operation": 0}, "__log": {"MenuID": "F71C97D5-D3E2-4FDA-9209-D7FA8626390E", "Logtype": 6, "Context": "查询"}}), headers=headers ) CurWeek = json.loads(r.content.decode('utf-8'))['data']['CurWeek'] r = self.__session.post( - url='http://'+self.__ip[0]+'-'+self.__ip[1]+'-'+self.__ip[2]+'-'+self.__ip[3] + - '-8080-p.webvpn.cust.edu.cn:8118/api/ClientStudent/Home/StudentHomeApi/QueryStudentScheduleData?sf_request_type=ajax', + url='https://jwgls1.cust.edu.cn/api/ClientStudent/Home/StudentHomeApi/QueryStudentScheduleData?sf_request_type=ajax', data=json.dumps({"param": "JTdCJTdE", "__permission": {"MenuID": "F71C97D5-D3E2-4FDA-9209-D7FA8626390E", "Operation": 0}, "__log": {"MenuID": "F71C97D5-D3E2-4FDA-9209-D7FA8626390E", "Logtype": 6, "Context": "查询"}}), headers=headers diff --git a/lib/crawler_test.py b/lib/crawler_test.py index d4ff8c0..dfad67b 100644 --- a/lib/crawler_test.py +++ b/lib/crawler_test.py @@ -8,18 +8,18 @@ class TestCrawler(unittest.TestCase): def test_connection(self): self.assertEqual(c.connection(), ('ok', 200)) - # 测试获取成绩 - # def test_grade(self): - # self.assertEqual(c.getGrade(), ('ok', 200)) + #测试获取成绩 + def test_grade(self): + self.assertEqual(c.getGrade(), ('ok', 200)) - # 测试获取课表 - # def test_schedule(self): - # self.assertEqual(c.getSchedule(), ('ok', 200)) + #测试获取课表 + def test_schedule(self): + self.assertEqual(c.getSchedule(), ('ok', 200)) - # 测试返回信息 - # def test_getData(self): - # get_res = c.getData() - # self.assertEqual(get_res['errcode'], '200') + #测试返回信息 + def test_getData(self): + get_res = c.getData() + self.assertEqual(get_res[1], 200) if __name__ == '__main__': unittest.main() diff --git a/lib/db.py b/lib/db.py index b2c3015..53dccbb 100644 --- a/lib/db.py +++ b/lib/db.py @@ -2,8 +2,8 @@ from pymongo import MongoClient from bson import ObjectId, json_util # 主环境 (生产环境为production，开发环境为development) -setting = 'development' -env = 'coc' +setting = 'production' +env = 'ali' # 获取数据集 @@ -29,7 +29,7 @@ def col(arg): def addRank(nick, count, time): try: col('rank').update({"cid": cid}, {'$setOnInsert': {"nick": nick}, '$set': { - "count": count, "time": time}}, {'upsert': 'true'}) + "count": count, "time": time}} , {'upsert': 'true'}) # col('rank').insert_one({"count":count,"time":time,"nick":nick}) except Exception as e: # 失败 @@ -37,8 +37,6 @@ def addRank(nick, count, time): return {'errcode': 200, 'errmsg': 'ok'} # 获取排名表所有信息（除了id） - - def getRank(): time_rank = [] count_rank = []