因为mitmproxy没办法连接数据库所以，只能先把结果保存为txt文件，再读取到数据库中。

在滑动APP界面时，对代码进行分析

import requestsimport reimport urllibdef requets(flow):    flow.request.headers['User-Agent'] = 'MitmProxy'    print(flow.request.headers)

保存到文本：

 

爬取程序

from mitmproxy import ctximport jsondef response(flow):    print('获取列表数据中...')    #url = 'http://mapi.dangdang.com/index.php?cat_path_text=%E6%80%BB%E6%A6%9C&img_size=b&bang_name_text=%E5%9B%BE%E4%B9%A6%E7%95%85%E9%94%80%E6%A6%9C&access-token=&permanent_id=20190405192112099238317917598184509&ischildren=0&bang_name=bestsell&user_client=android&pagesize=10&union_id=537-100380&time_code=849e796eaa9e66cae342dd1f47c5f032&action=bang_tushu&page=5&global_province_id=151&cat_path=01.00.00.00.00.00&client_version=9.4.2&udid=c906965ad731be703305409f738a1bad&timestamp=1556893369'    url = flow.request.url    if flow.request.url.startswith(url):        text = flow.response.text        #print(text)        data = json.loads(text)        books = data.get('products')        for book in books:            info = {                '书名': book.get('product_name'),                '作者': book.get('author'),                '价格': book.get('price').get('dangdang_price'),                '封面图片': book.get('img_url'),            }            ctx.log.info(str(book))            with open('D:\\books.txt', 'a', encoding='utf-8') as f:                f.write(json.dumps(info, ensure_ascii=False) + '\n')

 

 

结果

 

读取到数据库中

import pymongofrom pymongo import MongoClientimport jsonclient = pymongo.MongoClient('mongodb://admin:admin@localhost:27017/')db = client['books']collection = db['book']with open('D:\\books.txt', 'r+', encoding='utf-8') as f:    for i in f.readlines():        new = json.loads(i)        try:            if collection.insert(new):                print("成功保存到MongoDB")        except Exception:            print('someing wrong with MongDB')