Def save_stocks(): all_stocks = target_util._get_all_stocks() with open ("stock.csv",'a+') as f: F.rite (" stock symbol, stock name, market, Category, type \n") for stock in all_stocks: f.write("{stock[id]},{stock[name]},{stock[category]},{stock[tag]},{stock[type]}\n".format( stock=stock )) Logging.info (" All stock info written!" ) if __name__ == "__main__": save_stocks()Copy the code
Def _get_all_stocks(): base_url = "http://54.push2.eastmoney.com/api/qt/clist/get?pn={page_num}&pz={page_size}&po=1&np=1&fltt=2&invt=2&fid=f3&fs={time_id} & fields = f12, f14 "stocks = {" category" : "A shares", "tag" : "the Shanghai and shenzhen A shares", "type" : "stock", "time_id" : "M: 0 + t: 6, m: 0 + t: 80, m: t: 1 + 2, m: 1 + t: 23"}, {" category ":" A shares ", "tag" : "the Shanghai a-share market", "type" : "stock", "time_id" : "M: 1 + t: 2, m: 1 + t: 23"}, {" category ":" A shares ", "tag" : "the shenzhen A shares", "type" : "stock", "time_id" : "m: 0 + t: 6, m: 0 + t: 80"}, {" category ": "A shares" and "tag", "new", "type" : "stock", "time_id" : "m: f: 0 + 8, m: 1 + f: 8"}, {" category ":" A shares ", "tag" : "gem", "type" : "Stock", "time_id" : "m: 0 + t: 80"}, {" category ":" A shares ", "tag" : "kechuang board", "type" : "stock", "time_id" : "1 + m: t: 23"}, {" category ": "A shares" and "tag", "Shanghai tong", "type" : "stock", "time_id" : "b: BK0707"}, {" category ":" A shares ", "tag" : "deep stocks", "type" : "stock", "time_id" : "B: BK0804"}, {" category ":" b shares ", "tag" : "b shares", "type" : "stock", "time_id" : "m: 0 + t: 7, m: t: 1 + 3"}, {" category ": "A - B shares", "tag" : "the Shanghai AB stock price comparison", "type" : "stock", "time_id" : "1 + m: B: BK0498"}, {" category ":" A - B shares ", "tag" : "the shenzhen AB stock price", "type" : "Stock", "time_id" : "m: 0 + b: BK0498"}, {" category ":" A - b shares ", "tag" : "risk warning board", "type" : "stock", "time_id" : "M: f: 0 + 4, m: 1 + f: 4"}, {" category ":" A - B shares ", "tag" : "two nets and retreat city", "type" : "stock", "time_id" : "m: 0 + s: 3"}, {" category ": "Wall Street", "tag" : "Wall Street", "type" : "stock", "time_id" : "m: 105, m: 106, m: 107"}, {" category ":" Hong Kong ", "tag" : "Hong Kong", "type" : "Stock", "time_id" : "m: 128 + t: 3, m: 128 + t: 4, m: + 128 t: 1, m: 128 + t: 2"}, {" category ":" the stock ", "tag" : "the stock", "type" : "stock", "time_id" : "m:155+t:1,m:155+t:2,m:155+t:3,m:156+t:1,m:156+t:2,m:156+t:5,m:156+t:6,m:156+t:7,m:156+t:8" } ] all_stocks = [] for Stock in stocks: all_stocks.extend(_get_stocks(base_URL, stock)) logging.warning(" all stock information is {0}." .format(len(all_stocks))) return all_stocksCopy the code
Def _get_stocks(base_URL, stock): max_page_num = 50 page_size = 100 result = [] for page_num in range(1, max_page_num): url = base_url.format(time_id=stock["time_id"], page_num=page_num, page_size=page_size) resp = requests.get(url) if not resp.ok: Logging. error("{0}-{1}-{2} Request failed: {3}".format(stock["type"], stock["category"], stock["tag"], url)) resp_json = resp.json() if not resp_json["data"]: Logging. Warning (" no data on the current page, no further requests!" ) break stocks = resp_json["data"]["diff"] result.extend(list( map(lambda s: {"id": s["f12"].replace(" ", "").replace("'", "_"), "name": s["f14"].replace(" ", "").replace("'", "_"), "category": Stock [] "category", "tag" : stock/" tag ", "type" : stock "type"}, stocks))) logging. The info (" {0} {1} - {2} information crawl is complete, a total of {3} article." .format(stock["type"], stock["category"], stock["tag"], len(result))) return resultCopy the code