@nger/weibo

SELECT \"SpilderTopicArticleComment\".\"id\" AS \"SpilderTopicArticleComment_id\", \"SpilderTopicArticleComment\".\"platform\" AS \"SpilderTopicArticleComment_platform\", \"SpilderTopicArticleComment\".\"eid\" AS \"SpilderTopicArticleComment_eid\", \"SpilderTopicArticleComment\".\"tid\" AS \"SpilderTopicArticleComment_tid\", \"SpilderTopicArticleComment\".\"aid\" AS \"SpilderTopicArticleComment_aid\", \"SpilderTopicArticleComment\".\"created_at\" AS \"SpilderTopicArticleComment_created_at\", \"SpilderTopicArticleComment\".\"cid\" AS \"SpilderTopicArticleComment_cid\", \"SpilderTopicArticleComment\".\"floor_number\" AS \"SpilderTopicArticleComment_floor_number\", \"SpilderTopicArticleComment\".\"text\" AS \"SpilderTopicArticleComment_text\", \"SpilderTopicArticleComment\".\"source\" AS \"SpilderTopicArticleComment_source\", \"SpilderTopicArticleComment\".\"uid\" AS \"SpilderTopicArticleComment_uid\", \"SpilderTopicArticleComment\".\"nickname\" AS \"SpilderTopicArticleComment_nickname\", \"SpilderTopicArticleComment\".\"like_counts\" AS \"SpilderTopicArticleComment_like_counts\", \"SpilderTopicArticleComment\".\"total_number\" AS \"SpilderTopicArticleComment_total_number\", \"SpilderTopicArticleComment\".\"max_id\" AS \"SpilderTopicArticleComment_max_id\", \"SpilderTopicArticleComment\".\"is_liked_by_mblog_author\" AS \"SpilderTopicArticleComment_is_liked_by_mblog_author\", \"SpilderTopicArticleComment\".\"pid\" AS \"SpilderTopicArticleComment_pid\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"id\" AS \"8a653534da7d86b8b1e6ef6609f201284f2ec9747c4d0ae27588a5b56b33357\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"title\" AS \"27b2bcb998acf92b45dd981dfee2910ff779ac301ef532066ae93236d17afcc\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"name\" AS \"7f2fc58e5d58a3df4c3a956b9b89327fddede66a71e7a3d7a78f03ed95a705d\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"icon\" AS \"289f4ed7799837207cd46ecc3756cd08ccade7ad5df63380d5658bd1145c7da\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"loginUrl\" AS \"7ff7110e6ed098c9e98ec56b5937884197a0995c62d8e6fe48e8bcf0d69b1a2\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"uid\" AS \"6ef2989cbd19bb5e6a5ca3d5d49052e60b841f4c85686c77ec20e8849aaa525\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"nickname\" AS \"6f72bf0fca1a0f426cec2e1410d9ff7c0ed97df29f4c2da9ab98885e4576178\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"status\" AS \"8828f1c3e40aaac6e49611bcb8baed8123918121194a325369203f5d2909000\", \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"cookies\" AS \"17ba4f55d3ab2b7990e0aae938b2835a833ad32189208253d98d43c44932383\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"id\" AS \"SpilderTopicArticleComment__SpilderTopicArticleComment_event_id\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"title\" AS \"144c8b2a4b9abc7c56ca94cd5c5eaf6b80fb093798c02b435e680dab76a60b6\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"address\" AS \"7b8b57479967a79c6079daf5342d1e806b7a9b38780c6f695629567cebeb58b\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"type\" AS \"efe5c93bd5f70ee4a843529bf920e687acdfc29d8d553146838884017b9eb35\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"category\" AS \"4025d6ec905395a148bb38e3db0058b97a8ba589206de5bb98a5261ed941554\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"desc\" AS \"11ae734aa9f1ff0d5304763359317d88107bd8690bb37c91a0470e1e816967c\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"tid\" AS \"39f29b3de450a85a4013da74ed9fe556f69d1dc056324d61d8bc8525c2fe187\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"has_police\" AS \"c11638f301c445d46390a04de4fbf99b9fc56cf1aea375d19d68d76d912680a\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"handler\" AS \"0b6b39a0b571b9ab38a559226e8df1e3c0a9d7843ae2d3198e85aae6d00ce7a\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"keywords\" AS \"9a4419a735221c6846b15fc324476b1bec5f92ef6c8e2194ae35a700174122c\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"id\" AS \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic_id\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"platform\" AS \"f309ef6fc0c090e49a20cf4dcc315573a1b3d6ff643c5d49c1108c984529dad\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"title\" AS \"fbe9ec85e2b2b68b57f0a0c20391792805b4e8f0267bf318efa731e15d4efac\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"uid\" AS \"ae9b3d8b73cd878a5ca6bec8aa8eecc64de23391f3e5dee003746873470f6de\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"eid\" AS \"7432b5f7c2cb74fe8bb88770b637ada1b378f84cc9ec211c9927d8fe8189512\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"cate_id\" AS \"da32c39b5415a3068ab4195ae04932800bb35dca809a2fb3726db8c61fea1ee\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"category_str\" AS \"19bff8dcad426e54bf94436de6a98e2beef7be7458d652361018a1c971933ea\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"location\" AS \"fc85c9918870722d1c1f10986ac43ae1814d985e9c2591e8902e4c1cd891729\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"related_topic\" AS \"5bf9de939c9b68431278fd796c0d50c021f2509317905838a37b7031242c589\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"summary\" AS \"7ca10b250248d1983fdf613f107451c0bc48a54b1989429b739555a543b2da0\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"target_url\" AS \"b7a0e83902376adffe9fd87bc2d4cad042f497c27faa97c1cf7db3897b0f7aa\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"topic_tag\" AS \"f4aaaf031e33d94035bd8b4cf8df44a3aded29751182013cb243de1a0de3404\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"create_at\" AS \"7a4f7d33a08096aecc40c40e5c44e1dd7fa05a124589c022ad452230c38ad12\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"claim_id\" AS \"9398bc511b8db6e61d188f718c3e2974c13f2bafc0bdb442d75be3762367d9f\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"claim_name\" AS \"8b1b077b40b57a908e26e5cfcdf0daaee2c42db259bd8d1d0f7678afaeb9080\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"read_count\" AS \"2fb5429551890e5901c16d9d1940b0f8efd6d6a83f730826e40ec031502f255\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"mention_count\" AS \"7647eff0b9f0682d475a468b39ac7f316480da3b2e7aeea88fc9efd0021b9a8\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"ori_uv_Count\" AS \"c07fb8c83f768acdc7b4b9e54a30be6d8432a7a1ca4cae70a48aa0b12ba3756\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"id\" AS \"cc784e204216e7ce5e369f13ad43c3992d08c5838efe4a2c3e4d29e7e45ea00\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"tid\" AS \"e315f643797f666a89031adfeef9b550afd94d8b8599f9dac432490632d4117\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"eid\" AS \"2c1cf4d3abc37d29eef8d1cd51daf701c63082303559739c981c3365ca30d18\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"platform\" AS \"90221035aab986354ba3c0bc1113ea23f5c7dd5b106ddc978094c5d22835ecf\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"create_at\" AS \"c1ee7f04e7b178a5148bc61c15979ce01dca46e932a2057efd30706e758efba\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"mid\" AS \"cbd81c07c379f1351f074dbe11977fed154b8d88b3414c93ea5c94f5abe85b5\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"mblogid\" AS \"415169d28f4e2f6f112a54b9912a917f0990334ef2984dfab4d13b7da955ccd\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"title\" AS \"1f639e5dd4c66f583901e4d01c4a09509a76e069005119b9544f718799b32ff\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"uid\" AS \"714a267329c78b0157a466f488cd6c6029d80816e0724b96eda445c4db3857b\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"nickname\" AS \"734d1414910ea94f5e9775806c3cff74fc766e6dd06acd82a86b6648ea50038\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"text_raw\" AS \"b16fda24992c96f3e1eb5d3c94e4e6360bdcc77f3a047350b5504a495e8ea96\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"source\" AS \"1ddc64cded3709f0958d02358e6c2cd3c4f2ffb96bea39e13b7a78f7ae4028b\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"reposts_count\" AS \"84f6393e7228ef48bdce65ec2cc1256685e9f5fc8aa1aa7e4de399f8edccab2\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"comments_count\" AS \"b5655efc95fcb202b43bb56fcc969d954f0679bbe83186bfa2c71e691f2967f\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"attitudes_count\" AS \"be265e8aea4989ca5763c99ec7568a6ac24bb4f9216bc17c5749f5f3ff0235b\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"pic_bg_new\" AS \"f05dbff77c5b721090c8a29f723109da008f51f9261227295cc34613fe737cd\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"structs\" AS \"2da6c81ce404d65a754966dce1bcd663764a842e5b0d1c4847107e35bd3d74f\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"region_name\" AS \"861a570d9c6cf8e9066665c4ebae21e1b9461b7ed9771e763bbdc522002a6c9\", \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"text_length\" AS \"13ba4b1bcbb35802265f195ec768b2e74292c632ababc330fe09b107cf73bbc\" FROM \"spilder_topic_article_comment\" \"SpilderTopicArticleComment\" LEFT JOIN \"wb_platform\" \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\" ON \"b72134ae8d0c9703010b03069223490814bfe75f0a122faa68a9d6c70d94651\".\"name\"=\"SpilderTopicArticleComment\".\"platform\"  LEFT JOIN \"wb_event\" \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\" ON \"SpilderTopicArticleComment__SpilderTopicArticleComment_event\".\"id\"=\"SpilderTopicArticleComment\".\"eid\"  LEFT JOIN \"spilder_topic\" \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\" ON \"SpilderTopicArticleComment__SpilderTopicArticleComment_topic\".\"id\"=\"SpilderTopicArticleComment\".\"tid\"  LEFT JOIN \"spilder_topic_article\" \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\" ON \"SpilderTopicArticleComment__SpilderTopicArticleComment_article\".\"id\"=\"SpilderTopicArticleComment\".\"aid\" ORDER BY \"SpilderTopicArticleComment\".\"id\" DESC

1. spacy
   • 分词、词性标注、词干化、命名实体识别、名词短语提取

踩坑了

10.1.10.110 alma 123qwe

内网IP 192.168.2.180 （通过爬虫服务器访问） postgres 端口 15432 密码 pg123qwe rabbitmq 是 15672 管理 18080 guest、guest

任务设计

1. 话题分页任务

根据关键字，话题列表搜索，每个任务负责某一页话题数据抓取，每条话题创建《话题任务》

2. 话题详情任务--话题详情连接

根据连接，抓取话题详情并入库，并创建《帖子分页任务》

3. 帖子分页任务

根据连接，每个任务负责某一页帖子抓取，每个帖子创建《帖子任务》

4. 帖子任务

根据连接，抓取帖子详情并入库， 并创建《评论抓取任务》

5. 评论抓取任务

根据要求抓取评论及子评论

6. 用户信息抓取

交付内容：

1. 首批测试数据（157个事件数据）（按提供数据结构表爬取并整理）；
2. 第二批数据（事件数据）
3. 微博、知乎及其他一个平台爬虫源码
4. 一个数据查询、处理调取的平台系统

交付时间：

事件数据：

1. 首批测试数据：2022年 8月2日前交付微博平台，其余平台8月5日交付
2. 第二批数据（事件数据）：2022年8月17日前（待定）
3. 爬虫源码：2022年8月22日前
4. 两个爬虫平台系统：2022年8月30日前
5. 整体（三个）2022年9月30日前

工作计划

2022/7/13

1. 了解项目需求，确定技术方案及可行性
2. 按照需求，构建五层数据库结构，并完成主流媒体数据批量导入数据库
3. 商讨算法及模型处理办法-外包专业算法团队

2022/7/14

1. 完成微博扫码登录
2. 根据关键字 抓取主题

2022/7/15

1. 根据主题抓取评论/子评论
2. 字数统计

2022/7/16

1. 创建消息队列，构建任务池
2. 多节点执行抓取任务
3. 拿到部分测试数据，对接算法

2022/7/17

1. 事件的增删改查页面，根据关键字下发抓取主题任务
2. 根据主题，下发抓取评论任务

2022/7/18

1. 登录及账号功能
2. 用户增删改查

2022/7/19

1. UI界面美化
2. 服务器环境到位--建议

2022/7/20 - 2022/7/23

1. 培训讲解
2. 上线试运行
3. 协助算法执行结果，进行入库存储
4. 根据反馈进行操作流程调整

2022/7/24-29

1. 根据问题优化
2. BUG及问题修复
3. 完成基础数据采集工作

2022/7/30--2025/7/30

1. 持续优化