Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
Z
zzsn_spider
概览
概览
详情
活动
周期分析
版本库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
统计图
问题
0
议题
0
列表
看板
标记
里程碑
合并请求
1
合并请求
1
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
丁双波
zzsn_spider
Commits
ef3f6a6c
提交
ef3f6a6c
authored
9月 09, 2024
作者:
XveLingKun
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
0909-更新企业同步接口
上级
a4ae24a4
隐藏空白字符变更
内嵌
并排
正在显示
50 个修改的文件
包含
72 行增加
和
71 行删除
+72
-71
RedisPPData.py
base/RedisPPData.py
+1
-1
研究中心需更新企业.py
base/研究中心需更新企业.py
+2
-2
BaseCore.py
comData/NQenterprise/BaseCore.py
+1
-1
finance_xq.py
comData/NQenterprise/finance_xq.py
+2
-2
att_id.py
comData/YanBao/att_id.py
+1
-1
get_updateinfo.py
comData/YanBao/get_updateinfo.py
+1
-1
resentYanbao.py
comData/YanBao/resentYanbao.py
+2
-2
update_attid.py
comData/YanBao/update_attid.py
+1
-1
uploadfile.py
comData/YanBao/uploadfile.py
+1
-1
BaseCore.py
comData/annualReport1014/BaseCore.py
+1
-1
BaseCore.py
comData/annualReport1023/BaseCore.py
+1
-1
es_mysql_1.py
comData/annualReport1023/es_mysql_1.py
+1
-1
updateyear.py
comData/annualReport1023/updateyear.py
+1
-1
uptoes.py
comData/annualReport1023/uptoes.py
+1
-1
uploadfile.py
comData/annualReport_US/uploadfile.py
+1
-1
BaseCore.py
comData/caiwushuju/BaseCore.py
+1
-1
sina_usstock财务.py
comData/caiwushuju/sina_usstock财务.py
+4
-4
test2.py
comData/caiwushuju/test2.py
+3
-3
test3.py
comData/caiwushuju/test3.py
+1
-1
经营分析.py
comData/caiwushuju/经营分析.py
+3
-3
see_measures.py
comData/dingzhi/see_measures.py
+1
-1
nasdaq_caiwu.py
comData/nasdaq/nasdaq_caiwu.py
+3
-3
creditchina.py
comData/negative_news/creditchina.py
+1
-1
BaseCore.py
comData/newlist/champion/BaseCore.py
+1
-1
BaseCore.py
comData/newlist/china100/BaseCore.py
+1
-1
BaseCore.py
comData/newlist/global100/BaseCore.py
+1
-1
BaseCore.py
comData/newlist/hundred/BaseCore.py
+1
-1
BaseCore.py
comData/newlist/technological/BaseCore.py
+1
-1
BaseCore.py
comData/policylaw/BaseCore.py
+1
-1
deletedup.py
comData/policylaw/deletedup.py
+1
-1
BaseCore.py
comData/reits2/BaseCore.py
+1
-1
BaseCore.py
comData/tender/BaseCore.py
+1
-1
baseCore.py
comData/tradingview/baseCore.py
+1
-1
BaseCore.py
comData/tyctest/BaseCore.py
+1
-1
inputSql.py
comData/weixin_solo/inputSql.py
+1
-1
oneWeixin.py
comData/weixin_solo/oneWeixin.py
+2
-2
test.py
comData/weixin_solo/test.py
+1
-1
BaseCore.py
cpws/BaseCore.py
+1
-1
googletaskJob_loc.py
google_comm/googletaskJob_loc.py
+1
-1
BaseCore.py
gwzk/BaseCore.py
+2
-2
qiushi_leaderspeech.py
qiushi_leaderspeech.py
+1
-1
sclx.py
shenji/sclx.py
+1
-1
write_tocsv.py
shenji/write_tocsv.py
+1
-1
BaseCore.py
tmp/usVsRussia/BaseCore.py
+1
-1
aaaa.py
习近平讲话/aaaa.py
+2
-2
datasfromes.py
习近平讲话/datasfromes.py
+2
-2
update_title.py
习近平讲话/update_title.py
+2
-2
write_tocsv.py
习近平讲话/write_tocsv.py
+2
-2
baidutaskJob_loc.py
百度采集/baidu_comm/baidutaskJob_loc.py
+2
-2
baseCore.py
百度采集/baidu_comm/baseCore.py
+4
-3
没有找到文件。
base/RedisPPData.py
浏览文件 @
ef3f6a6c
...
...
@@ -20,7 +20,7 @@ def close11(cnx_,cursor_):
cursor_
.
close
()
# # 连接到Redis
# r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn', db=6)
# r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn', db=6)
#
# cnx = pymysql.connect(host='1.95.78.131', user='caiji', password='zzsn9988', db='caiji',
# charset='utf8mb4')
...
...
base/研究中心需更新企业.py
浏览文件 @
ef3f6a6c
from
apscheduler.schedulers.blocking
import
BlockingScheduler
from
apscheduler.schedulers.blocking
import
BlockingScheduler
...
...
@@ -29,7 +29,7 @@ def putCom():
df
=
pd
.
read_excel
(
'D:
\\
企业数据
\\
数据组提供
\\
国内企业.xlsx'
)
# 连接到Redis数据库
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
for
i
in
range
(
len
(
df
)):
social_code
=
df
[
'social_code'
][
i
]
com_name
=
df
[
'name'
][
i
]
...
...
comData/NQenterprise/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -251,7 +251,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/NQenterprise/finance_xq.py
浏览文件 @
ef3f6a6c
...
...
@@ -28,7 +28,7 @@ def check_code(com_code):
return
True
def
check_date
(
com_code
,
info_date
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
sismember
(
'com_xqcwsj_code::'
+
com_code
,
info_date
)
# 注意是 保存set的方式
if
res
:
return
True
...
...
@@ -37,7 +37,7 @@ def check_date(com_code,info_date):
# 将采集后的股票代码对应的报告期保存进redis
def
add_date
(
com_code
,
report_date
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
sadd
(
'com_xqcwsj_code::'
+
com_code
,
report_date
)
...
...
comData/YanBao/att_id.py
浏览文件 @
ef3f6a6c
...
...
@@ -26,7 +26,7 @@ lock = threading.Lock()
pathType
=
'QYNotice/'
taskType
=
'企业研报/东方财富网'
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
class
EsMethod
(
object
):
...
...
comData/YanBao/get_updateinfo.py
浏览文件 @
ef3f6a6c
...
...
@@ -29,7 +29,7 @@ lock = threading.Lock()
pathType
=
'QYNotice/'
taskType
=
'企业研报/东方财富网'
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
class
EsMethod
(
object
):
...
...
comData/YanBao/resentYanbao.py
浏览文件 @
ef3f6a6c
...
...
@@ -24,7 +24,7 @@ sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import
BaseCore
baseCore
=
BaseCore
.
BaseCore
()
log
=
baseCore
.
getLogger
()
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
)
obsClient
=
ObsClient
(
access_key_id
=
'VEHN7D0TJ9316H8AHCAV'
,
# 你的华为云的ak码
secret_access_key
=
'heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY'
,
# 你的华为云的sk
...
...
@@ -119,7 +119,7 @@ def add_check_url(article_url):
# redis上传失败删除数据
def
delete_url
(
article_url
):
# r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn')
# r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn')
res
=
r
.
srem
(
'report_pdf_three_history'
,
article_url
)
if
res
>
0
:
return
True
...
...
comData/YanBao/update_attid.py
浏览文件 @
ef3f6a6c
...
...
@@ -21,7 +21,7 @@ pathType = 'QYNotice/'
taskType
=
'企业研报/东方财富网'
es
=
Elasticsearch
([
'http://1.95.3.121:9200'
],
http_auth
=
(
'elastic'
,
'zzsn9988'
),
timeout
=
300
)
index_name
=
'researchreportdata'
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
def
searchATT
():
sql
=
"select id from clb_sys_attachment where type_id=4 and create_time>'2023-12-08' "
...
...
comData/YanBao/uploadfile.py
浏览文件 @
ef3f6a6c
...
...
@@ -25,7 +25,7 @@ lock = threading.Lock()
pathType_
=
'QYResearchReport/'
taskType
=
'企业研报/东方财富网'
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
obsClient
=
ObsClient
(
access_key_id
=
'VEHN7D0TJ9316H8AHCAV'
,
# 你的华为云的ak码
...
...
comData/annualReport1014/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -263,7 +263,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/annualReport1023/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -261,7 +261,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/annualReport1023/es_mysql_1.py
浏览文件 @
ef3f6a6c
...
...
@@ -26,7 +26,7 @@ lock = threading.Lock()
pathType
=
'QYNotice/'
taskType
=
'企业研报/东方财富网'
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
class
EsMethod
(
object
):
...
...
comData/annualReport1023/updateyear.py
浏览文件 @
ef3f6a6c
...
...
@@ -25,7 +25,7 @@ lock = threading.Lock()
pathType_
=
'QYResearchReport/'
taskType
=
'企业研报/东方财富网'
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
obsClient
=
ObsClient
(
access_key_id
=
'VEHN7D0TJ9316H8AHCAV'
,
# 你的华为云的ak码
...
...
comData/annualReport1023/uptoes.py
浏览文件 @
ef3f6a6c
...
...
@@ -20,7 +20,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
baseCore
=
BaseCore
.
BaseCore
()
log
=
baseCore
.
getLogger
()
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
cnx
=
pymysql
.
connect
(
host
=
'114.116.44.11'
,
user
=
'caiji'
,
password
=
'f7s0&7qqtK'
,
db
=
'clb_project'
,
charset
=
'utf8mb4'
)
cursor_
=
cnx
.
cursor
()
lock
=
threading
.
Lock
()
...
...
comData/annualReport_US/uploadfile.py
浏览文件 @
ef3f6a6c
...
...
@@ -26,7 +26,7 @@ cursor_ = cnx_.cursor()
lock
=
threading
.
Lock
()
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
class
EsMethod
(
object
):
...
...
comData/caiwushuju/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -230,7 +230,7 @@ class BaseCore:
self
.
cursor
=
self
.
cnx
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
def
close
(
self
):
try
:
...
...
comData/caiwushuju/sina_usstock财务.py
浏览文件 @
ef3f6a6c
impor
t
configparser
impor
t
configparser
...
...
@@ -80,7 +80,7 @@ class SinaUsstock(object):
# 判断股票代码是否存在
def
check_code
(
self
,
com_code
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
exists
(
'com_sinacaiwushuju_code::'
+
com_code
)
#如果key存在 则不是第一次采集该企业, res = 1
if
res
:
...
...
@@ -89,7 +89,7 @@ class SinaUsstock(object):
return
True
#表示是第一次采集
def
check_date
(
self
,
com_code
,
info_date
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
sismember
(
'com_sinacaiwushuju_code::'
+
com_code
,
info_date
)
# 注意是 保存set的方式
if
res
:
return
True
...
...
@@ -98,7 +98,7 @@ class SinaUsstock(object):
# 将采集后的股票代码对应的报告期保存进redis
def
add_date
(
self
,
com_code
,
date_list
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
#遍历date_list 放入redis
for
date
in
date_list
:
res
=
r
.
sadd
(
'com_sinacaiwushuju_code::'
+
com_code
,
date
)
...
...
comData/caiwushuju/test2.py
浏览文件 @
ef3f6a6c
import
redis
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
# 获取所有键
# keys = r.keys('*')
# # print(keys)
...
...
@@ -11,7 +11,7 @@ r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
# r.srem(f_key,'[]')
# r.srem(f_key,'')
def
check_date
(
com_code
,
info_date
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
sismember
(
'com_caiwushuju_code::'
+
com_code
,
info_date
)
# 注意是 保存set的方式
if
res
:
return
True
...
...
@@ -20,7 +20,7 @@ def check_date(com_code,info_date):
def
add_date
():
date_list
=
[
'2023-06-30'
]
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
#遍历date_list 放入redis
for
date
in
date_list
:
res
=
r
.
sadd
(
'com_caiwushuju_code::'
+
'123456'
,
date
)
...
...
comData/caiwushuju/test3.py
浏览文件 @
ef3f6a6c
...
...
@@ -3,7 +3,7 @@
import
redis
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
f_key
=
'com_caiwushuju_code::bj851834'
r
.
set
(
f_key
,
3
)
res
=
r
.
exists
(
f_key
)
...
...
comData/caiwushuju/经营分析.py
浏览文件 @
ef3f6a6c
impor
t
pymysql
impor
t
pymysql
...
...
@@ -23,12 +23,12 @@ cursor= cnx.cursor()
# 将采集后的股票代码对应的报告期保存进redis
# def add_date(com_code,i):
# r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn',db=3)
# r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn',db=3)
# # json_cwsj = json.dumps(dic_cwsj)
# res = r.sadd('com_jingyingfenxi_code::'+com_code, i) # 注意是 保存set的方式
#
# def get_i():
# r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn', db=3)
# r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn', db=3)
# i = r.get('com_jingyingfenxi_code::'+com_code)
def
operate_analysis
(
i
,
com_code
,
social_code
):
...
...
comData/dingzhi/see_measures.py
浏览文件 @
ef3f6a6c
...
...
@@ -17,7 +17,7 @@ sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import
BaseCore
baseCore
=
BaseCore
.
BaseCore
()
log
=
baseCore
.
getLogger
()
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
5
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
5
)
# cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
cnx
=
baseCore
.
cnx_
obsClient
=
ObsClient
(
...
...
comData/nasdaq/nasdaq_caiwu.py
浏览文件 @
ef3f6a6c
...
...
@@ -28,7 +28,7 @@ taskType = '财务数据/纳斯达克'
# 判断股票代码是否存在
@retry
(
tries
=
3
,
delay
=
1
)
def
check_code
(
com_code
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
exists
(
'caiwu_nasdaq_code::'
+
com_code
)
# 如果key存在 则不是第一次采集该企业, res = 1
if
res
:
...
...
@@ -40,7 +40,7 @@ def check_code(com_code):
# 判断采集日期是否存在
@retry
(
tries
=
3
,
delay
=
1
)
def
check_date
(
com_code
,
info_date
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
res
=
r
.
sismember
(
'caiwu_nasdaq_code::'
+
com_code
,
info_date
)
# 注意是 保存set的方式
if
res
:
return
True
...
...
@@ -51,7 +51,7 @@ def check_date(com_code, info_date):
# 将采集后的股票代码对应的报告期保存进redis
@retry
(
tries
=
3
,
delay
=
1
)
def
add_date
(
com_code
,
date_list
):
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
3
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
3
)
# 遍历date_list 放入redis
for
date
in
date_list
:
res
=
r
.
sadd
(
'caiwu_nasdaq_code::'
+
com_code
,
date
)
...
...
comData/negative_news/creditchina.py
浏览文件 @
ef3f6a6c
...
...
@@ -28,7 +28,7 @@ baseCore = BaseCore()
log
=
baseCore
.
getLogger
()
cnx
=
baseCore
.
cnx
cursor
=
baseCore
.
cursor
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
0
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
0
)
import
urllib3
urllib3
.
disable_warnings
(
urllib3
.
exceptions
.
InsecureRequestWarning
)
taskType
=
'企业负面新闻'
...
...
comData/newlist/champion/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -251,7 +251,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/newlist/china100/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -246,7 +246,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/newlist/global100/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -246,7 +246,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/newlist/hundred/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -251,7 +251,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/newlist/technological/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -251,7 +251,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/policylaw/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -252,7 +252,7 @@ class BaseCore:
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/policylaw/deletedup.py
浏览文件 @
ef3f6a6c
...
...
@@ -21,7 +21,7 @@ cursor_ = cnx_.cursor()
lock
=
threading
.
Lock
()
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
pool
=
redis
.
ConnectionPool
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
class
EsMethod
(
object
):
def
__init__
(
self
):
...
...
comData/reits2/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -251,7 +251,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/tender/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -251,7 +251,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/tradingview/baseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -221,7 +221,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
__cursor_proxy
=
self
.
__cnx_proxy
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
# 计算耗时
def
getTimeCost
(
self
,
start
,
end
):
...
...
comData/tyctest/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -263,7 +263,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
comData/weixin_solo/inputSql.py
浏览文件 @
ef3f6a6c
...
...
@@ -5,7 +5,7 @@ baseCore = BaseCore()
log
=
baseCore
.
getLogger
()
cnx
=
baseCore
.
cnx
cursor
=
baseCore
.
cursor
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
)
#获取所有键
keys
=
r
.
keys
(
'*'
)
...
...
comData/weixin_solo/oneWeixin.py
浏览文件 @
ef3f6a6c
...
...
@@ -26,7 +26,7 @@ r = baseCore.r
urllib3
.
disable_warnings
()
def
check_url
(
sid
,
article_url
):
# r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn')
# r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn')
res
=
r
.
sismember
(
f
'wx_url_{sid}'
,
article_url
)
if
res
==
True
:
return
True
...
...
@@ -34,7 +34,7 @@ def check_url(sid, article_url):
return
False
def
add_url
(
sid
,
article_url
):
# r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn')
# r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn')
res
=
r
.
sadd
(
f
'wx_url_{sid}'
,
article_url
,
3
)
# 注意是 保存set的方式
if
res
==
0
:
# 若返回0,说明插入不成功,表示有重复
return
True
...
...
comData/weixin_solo/test.py
浏览文件 @
ef3f6a6c
...
...
@@ -47,7 +47,7 @@ from bs4 import BeautifulSoup
def
check_url
():
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
res
=
r
.
exists
(
'WeiXinGZH:infoSourceCode'
,
'IN-20220418-0001'
)
print
(
res
)
if
res
==
1
:
...
...
cpws/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -262,7 +262,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
google_comm/googletaskJob_loc.py
浏览文件 @
ef3f6a6c
...
...
@@ -192,7 +192,7 @@ if __name__ == '__main__':
# try:
# googleTaskJob.r.ping()
# except:
# googleTaskJob.r = redis.Redis(host="114.11
5.236.206", port=6379
, password='clbzzsn', db=6)
# googleTaskJob.r = redis.Redis(host="114.11
6.90.53", port=6380
, password='clbzzsn', db=6)
# all_keys = 'GOOGLE_KEYWORDS:COMPANY_NAME*'
# keys = googleTaskJob.r.scan_iter(f"{key}*")
# for key in keys:
...
...
gwzk/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -475,7 +475,7 @@ class BaseCore:
try
:
self
.
r
.
ping
()
except
:
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
item
=
self
.
r
.
lpop
(
key
)
return
item
.
decode
()
if
item
else
None
...
...
@@ -705,7 +705,7 @@ class BaseCore:
try
:
self
.
r
.
ping
()
except
:
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
.
rpush
(
key
,
item
)
# 增加计数器的值并返回增加后的值
...
...
qiushi_leaderspeech.py
浏览文件 @
ef3f6a6c
...
...
@@ -13,7 +13,7 @@ from base.BaseCore import BaseCore
baseCore
=
BaseCore
()
log
=
baseCore
.
getLogger
()
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
0
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
0
)
def
sendKafka
(
dic_news
):
try
:
producer
=
KafkaProducer
(
bootstrap_servers
=
[
'1.95.3.121:9092'
],
max_request_size
=
1024
*
1024
*
20
)
...
...
shenji/sclx.py
浏览文件 @
ef3f6a6c
...
...
@@ -17,7 +17,7 @@ from selenium.webdriver.support import expected_conditions as EC
urllib3
.
disable_warnings
()
baseCore
=
BaseCore
.
BaseCore
()
log
=
baseCore
.
getLogger
()
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
headers
=
{
'Accept'
:
'*/*'
,
'Accept-Encoding'
:
'gzip, deflate, br'
,
...
...
shenji/write_tocsv.py
浏览文件 @
ef3f6a6c
...
...
@@ -2,7 +2,7 @@ import csv
import
redis
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
if
__name__
==
"__main__"
:
with
open
(
'./output0220_1.csv'
,
'w'
,
newline
=
''
,
encoding
=
'utf-8'
)
as
file
:
writer
=
csv
.
writer
(
file
)
...
...
tmp/usVsRussia/BaseCore.py
浏览文件 @
ef3f6a6c
...
...
@@ -246,7 +246,7 @@ class BaseCore:
charset
=
'utf8mb4'
)
self
.
cursor_
=
self
.
cnx_
.
cursor
()
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
pool_caiji
=
PooledDB
(
creator
=
pymysql
,
...
...
习近平讲话/aaaa.py
浏览文件 @
ef3f6a6c
"""
"""
...
...
@@ -77,7 +77,7 @@ class EsMethod(object):
if
__name__
==
'__main__'
:
es_method
=
EsMethod
()
# 连接Redis
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
result
=
es_method
.
queryatt
(
'researchreportdata'
)
total
=
result
[
'hits'
][
'total'
][
'value'
]
...
...
习近平讲话/datasfromes.py
浏览文件 @
ef3f6a6c
"""
"""
...
...
@@ -53,7 +53,7 @@ class EsMethod(object):
if
__name__
==
'__main__'
:
es_method
=
EsMethod
()
# 连接Redis
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
for
i
in
range
(
56
):
result
=
es_method
.
queryatt
(
'subjectdatabase'
,
i
*
200
)
total
=
result
[
'hits'
][
'total'
][
'value'
]
...
...
习近平讲话/update_title.py
浏览文件 @
ef3f6a6c
impor
t
time
impor
t
time
...
...
@@ -35,7 +35,7 @@ def updateaunn(index_name,id,u_title):
,
body
=
body
)
print
(
'更新结果:
%
s'
%
result
)
if
__name__
==
'__main__'
:
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
headers
=
{
'Proxy-Connection'
:
'keep-alive'
,
'Accept'
:
'*/*'
,
...
...
习近平讲话/write_tocsv.py
浏览文件 @
ef3f6a6c
impor
t
csv
impor
t
csv
...
...
@@ -2,7 +2,7 @@ import csv
import
redis
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
if
__name__
==
"__main__"
:
with
open
(
'./title1.csv'
,
'w'
,
newline
=
''
,
encoding
=
'utf-8'
)
as
file
:
writer
=
csv
.
writer
(
file
)
...
...
百度采集/baidu_comm/baidutaskJob_loc.py
浏览文件 @
ef3f6a6c
# -*-
coding: utf-8 -*-
# -*-
coding: utf-8 -*-
...
...
@@ -207,7 +207,7 @@ def redicPullData(key, r):
try
:
r
.
ping
()
except
:
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
item
=
r
.
lpop
(
key
)
return
item
.
decode
()
if
item
else
None
...
...
百度采集/baidu_comm/baseCore.py
浏览文件 @
ef3f6a6c
# -*-
coding: utf-8 -*-
# -*-
coding: utf-8 -*-
...
...
@@ -217,7 +217,7 @@ class BaseCore:
pass
def
__init__
(
self
):
# 连接到Redis
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
__cnx_proxy
=
pymysql
.
connect
(
host
=
'1.95.78.131'
,
user
=
'caiji'
,
password
=
'zzsn9988'
,
db
=
'clb_project'
,
charset
=
'utf8mb4'
)
self
.
__cursor_proxy
=
self
.
__cnx_proxy
.
cursor
()
...
...
@@ -296,6 +296,6 @@ class BaseCore:
try
:
self
.
r
.
ping
()
except
:
self
.
r
=
redis
.
Redis
(
host
=
"114.11
5.236.206"
,
port
=
6379
,
password
=
'clbzzsn'
,
db
=
6
)
self
.
r
=
redis
.
Redis
(
host
=
"114.11
6.90.53"
,
port
=
6380
,
password
=
'clbzzsn'
,
db
=
6
)
item
=
self
.
r
.
lpop
(
key
)
return
item
.
decode
()
if
item
else
None
\ No newline at end of file
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论