Elasticsearch分页查询

global index
global CLIENT
index = "guajibao-ipused-2019.10.13"
CLIENT = Elasticsearch(hosts=es_host,port=port,timeout=timeout)

def load_all_data(scroll_id):
    if scroll_id:
        page = CLIENT.scroll(scroll_id=scroll_id, scroll='10m')
        scroll_id = page['_scroll_id']
    else:
        page = CLIENT.search(index=index, scroll='10m', size=100)
        scroll_id = page['_scroll_id']
    print(page)
    data = page["hits"]['hits']
    return {"data": data, "scroll_id": scroll_id, "length": len(data)}

例子

def load_all_data(scroll_id):
    if scroll_id:
        page = CLIENT.scroll(scroll_id=scroll_id, scroll='10m')
        scroll_id = page['_scroll_id']
    else:
        page = CLIENT.search(index=index, scroll='10m', size=100)
        scroll_id = page['_scroll_id']
    # print(page)
    data = page["hits"]['hits']
    total = page['hits']['total']['value']
    print(data)
    return  {"data": data, "scroll_id": scroll_id, "length": len(data), "total": total}

if __name__ == "__main__":
    result = load_all_data(None)
    total =  result['total']
    scroll_id = result['scroll_id']
    for i in range(int(total/100)):
        scroll_id = load_all_data(scroll_id)['scroll_id']

公告