Merge pull request #431 from solonobita/master

稳定性优化

depdencies.txt

@@ -0,0 +1,135 @@
+Python 3.8.10
+Package                 Version
+----------------------- --------------------
+absl-py                 1.2.0
+appdirs                 1.4.3
+astunparse              1.6.3
+attrs                   19.3.0
+Automat                 0.8.0
+autopep8                1.7.0
+beautifulsoup4          4.7.0
+blinker                 1.4
+browser-cookie3         0.16.1
+bs4                     0.0.1
+cachetools              4.2.4
+certifi                 2021.5.30
+chardet                 3.0.4
+charset-normalizer      2.1.1
+Click                   7.0
+cloud-init              22.2
+colorama                0.4.3
+command-not-found       0.3
+configobj               5.0.6
+constantly              15.1.0
+cryptography            2.8
+cssselect               1.0.3
+dbus-python             1.2.16
+DingtalkChatbot         1.3.0
+distro                  1.4.0
+distro-info             0.23ubuntu1
+entrypoints             0.3
+fake-useragent          0.1.11
+Flask                   1.0.2
+Flask-JWT-Extended      3.15.0
+gast                    0.3.3
+google-auth             1.35.0
+google-auth-oauthlib    0.4.6
+google-pasta            0.2.0
+grpcio                  1.48.1
+h5py                    2.10.0
+httplib2                0.14.0
+hyperlink               19.0.0
+idna                    2.8
+importlib-metadata      4.12.0
+incremental             16.10.1
+itsdangerous            1.1.0
+Jinja2                  2.10
+jsonpatch               1.22
+jsonpointer             2.0
+jsonschema              3.2.0
+Keras                   2.4.0
+Keras-Preprocessing     1.1.2
+keyring                 18.0.1
+language-selector       0.1
+launchpadlib            1.10.13
+lazr.restfulclient      0.14.2
+lazr.uri                1.0.3
+lightpush               0.1.3
+lxml                    4.6.3
+lz4                     4.0.2
+Markdown                3.4.1
+MarkupSafe              2.0.1
+more-itertools          4.2.0
+netifaces               0.10.4
+numpy                   1.18.5
+oauthlib                3.1.0
+opencv-python           4.6.0.66
+opt-einsum              3.3.0
+p5py                    1.0.0
+parse                   1.9.0
+pbkdf2                  1.3
+pbr                     5.10.0
+pep517                  0.13.0
+pexpect                 4.6.0
+pip                     22.2.2
+protobuf                3.9.2
+pyaes                   1.6.1
+pyasn1                  0.4.2
+pyasn1-modules          0.2.1
+pycodestyle             2.9.1
+pycryptodome            3.15.0
+pyee                    6.0.0
+PyGObject               3.36.0
+PyHamcrest              1.9.0
+PyJWT                   1.7.1
+pymacaroons             0.13.0
+PyNaCl                  1.3.0
+pyOpenSSL               19.0.0
+pypng                   0.20220715.0
+pyppeteer               0.0.25
+pyppeteer-box           0.0.27
+pyquery                 1.4.0
+pyrsistent              0.15.5
+pyserial                3.4
+python-apt              2.0.0+ubuntu0.20.4.8
+PyYAML                  5.3.1
+redis                   3.0.1
+requests                2.28.1
+requests-html           0.9.0
+requests-oauthlib       1.3.1
+requests-unixsocket     0.2.0
+rsa                     4.9
+scipy                   1.4.1
+SecretStorage           2.3.1
+service-identity        18.1.0
+setuptools              65.3.0
+simplejson              3.16.0
+six                     1.15.0
+sos                     4.3
+soupsieve               1.6.2
+ssh-import-id           5.10
+systemd-python          234
+tensorboard             2.10.0
+tensorboard-data-server 0.6.1
+tensorboard-plugin-wit  1.8.1
+tensorflow              2.3.0
+tensorflow-estimator    2.3.0
+termcolor               2.0.1
+testresources           2.0.1
+toml                    0.10.2
+tomli                   2.0.1
+tqdm                    4.64.1
+Twisted                 18.9.0
+typing_extensions       4.3.0
+ubuntu-advantage-tools  27.10
+ufw                     0.36
+unattended-upgrades     0.1
+urllib3                 1.26.12
+w3lib                   1.19.0
+wadllib                 1.3.3
+websockets              7.0
+Werkzeug                0.15.5
+wheel                   0.37.1
+wrapt                   1.14.1
+zipp                    1.0.0
+zope.interface          4.7.1

env.py.example

@@ -22,6 +22,9 @@ USER_ACCOUNTS = [
 # 接受字典形式 格式:    {'min': 0.5, 'max': 1}
 QUERY_INTERVAL = 1
 
+# 网络请求重试次数
+REQUEST_MAX_RETRY = 5
+
 # 用户心跳检测间隔 格式同上
 USER_HEARTBEAT_INTERVAL = 120
 

py12306/config.py

@@ -16,6 +16,8 @@ class Config:
     QUERY_JOBS = []
     # 查询间隔
     QUERY_INTERVAL = 1
+    # 查询重试次数
+    REQUEST_MAX_RETRY = 5
     # 用户心跳检测间隔
     USER_HEARTBEAT_INTERVAL = 120
     # 多线程查询

py12306/helpers/func.py

@@ -121,6 +121,8 @@ def str_to_time(str):
 def time_int():
     return int(time.time())
 
+def time_int_ms():
+    return int(time.time() * 1000)
 
 def is_number(val):
     if isinstance(val, int): return val

py12306/log/user_log.py

@@ -23,6 +23,7 @@ class UserLog(BaseLog):
     MESSAGE_USER_HEARTBEAT_NORMAL = '用户 {} 心跳正常，下次检测 {} 秒后'
 
     MESSAGE_GET_USER_PASSENGERS_FAIL = '获取用户乘客列表失败，错误原因: {} {} 秒后重试'
+    MESSAGE_TEST_GET_USER_PASSENGERS_FAIL = '测试获取用户乘客列表失败，错误原因: {} {} 秒后重试'
     MESSAGE_USER_PASSENGERS_IS_INVALID = '乘客信息校验失败，在账号 {} 中未找到该乘客: {}\n'
 
     # MESSAGE_WAIT_USER_INIT_COMPLETE = '未找到可用账号或用户正在初始化，{} 秒后重试'

py12306/order/order.py

@@ -24,6 +24,8 @@ def __init__(self, rect: dict) -> None:
 
 @singleton
 class Browser:
+    cookies = None
+    post_data = None
 
     def __init__(self) -> None:
         super().__init__()
@@ -79,6 +81,72 @@ async def on_request(req):
         await browser.close()
         return json.loads(ret)
 
+    def request_init_slide2(self, session, data):
+        """ 处理滑块，拿到 session_id, sig """
+        OrderLog.add_quick_log('正在识别滑动验证码...').flush()
+        return asyncio.get_event_loop_policy().new_event_loop().run_until_complete(
+            self.__request_init_slide2(data))
+
+    async def __request_init_slide2(self, data):
+        from random import randint
+        """ 异步获取 """
+        try:
+            browser = await launch(headless=True, autoClose=True, handleSIGINT=False, handleSIGTERM=False,
+                                   handleSIGHUP=False, args=['--disable-infobars', '--no-sandbox'])
+            page = await browser.newPage()
+            await page.setViewport({'width': 1200, 'height': 1080})
+            await page.setRequestInterception(True)
+            load_js = """() => {
+                __old = navigator.userAgent; navigator.__defineGetter__('userAgent', () => __old.replace('Headless', ''));
+                __old = navigator.appVersion; navigator.__defineGetter__('appVersion', () => __old.replace('Headless', ''));
+                var __newProto = navigator.__proto__; delete __newProto.webdriver; navigator.__proto__ = __newProto;
+            }"""
+
+            @page.on('framenavigated')
+            async def on_frame_navigated(_):
+                await page.evaluate(load_js)
+
+            @page.on('request')
+            async def on_request(req):
+                if req.url == 'https://kyfw.12306.cn/passport/web/login':
+                    self.post_data = req.postData
+                    return await req.abort()
+                return await req.continue_()
+
+            await page.goto('https://kyfw.12306.cn/otn/resources/login.html', timeout=30000)
+            await page.waitForSelector('#J-userName', timeout=30000)
+            await page.type('#J-userName', data['username'], {'delay': randint(10, 30)})  # 账号
+            await page.waitForSelector('#J-password', timeout=30000)
+            await page.type('#J-password', data['password'], {'delay': randint(10, 30)})  # 密码
+            await page.waitForSelector('#J-login', timeout=30000)
+            await page.focus('#J-login')
+            await page.click('#J-login')
+
+            slide_btn = await page.waitForSelector('#nc_1_n1z', timeout=30000)
+            rect = await slide_btn.boundingBox()
+            pos = DomBounding(rect)
+            pos.x += pos.width / 2
+            pos.y += pos.height / 2
+            await page.mouse.move(pos.x, pos.y)
+            await page.mouse.down()
+            await page.mouse.move(pos.x + pos.width * 10, pos.y, steps=30)
+            await page.mouse.up()
+            await page.evaluate(
+            'async () => {let i = 3 * 10; while (!csessionid && i >= 0) await new Promise(resolve => setTimeout(resolve, 100), i--);}')
+            await page.evaluate('JSON.stringify({session_id: csessionid, sig: sig})')
+            self.cookies = await page.cookies()
+            OrderLog.add_quick_log('滑动验证码识别成功').flush()
+        except Exception as e:
+            OrderLog.add_quick_log('滑动验证码识别失败').flush()
+        try:
+            await page.close()
+        except:
+            pass
+        try:
+            await browser.close()
+        except:
+            pass
+        return self.cookies, self.post_data
 
 class Order:
     """

py12306/query/query.py

@@ -119,11 +119,16 @@ def init_job(self, job):
         self.jobs.append(job)
         return job
 
-    def request_device_id(self):
+    def request_device_id(self, force_renew = False):
         """
         获取加密后的浏览器特征 ID
         :return:
         """
+        expire_time =  self.session.cookies.get('RAIL_EXPIRATION')
+        if not force_renew and expire_time and int(expire_time) - time_int_ms() > 0:
+            return
+        if 'pjialin' not in API_GET_BROWSER_DEVICE_ID:
+            return self.request_device_id2()
         response = self.session.get(API_GET_BROWSER_DEVICE_ID)
         if response.status_code == 200:
             try:
@@ -143,7 +148,35 @@ def request_device_id(self):
                         'RAIL_DEVICEID': Config().RAIL_DEVICEID,
                     })
             except:
-                return False
+                return self.request_device_id()
+        else:
+            return self.request_device_id()
+
+    def request_device_id2(self):
+        headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36"
+        }
+        self.session.headers.update(headers)
+        response = self.session.get(API_GET_BROWSER_DEVICE_ID)
+        if response.status_code == 200:
+            try:
+                if response.text.find('callbackFunction') >= 0:
+                    result = response.text[18:-2]
+                    result = json.loads(result)
+                    if not Config().is_cache_rail_id_enabled():
+                       self.session.cookies.update({
+                           'RAIL_EXPIRATION': result.get('exp'),
+                           'RAIL_DEVICEID': result.get('dfp'),
+                       })
+                    else:
+                       self.session.cookies.update({
+                           'RAIL_EXPIRATION': Config().RAIL_EXPIRATION,
+                           'RAIL_DEVICEID': Config().RAIL_DEVICEID,
+                       })
+            except:
+                return self.request_device_id2()
+        else:
+            return self.request_device_id2()
 
     @classmethod
     def wait_for_ready(cls):
@@ -180,11 +213,12 @@ def get_query_api_type(cls):
             res = re.search(r'var CLeftTicketUrl = \'(.*)\';', response.text)
             try:
                 self.api_type = res.group(1)
-            except IndexError:
+            except Exception:
                 pass
         if not self.api_type:
             QueryLog.add_quick_log('查询地址获取失败, 正在重新获取...').flush()
-            sleep(1)
+            sleep(get_interval_num(self.interval))
+        self.request_device_id(True)
         return cls.get_query_api_type()
 
 # def get_jobs_from_cluster(self):