提交 ef3f6a6c 作者: XveLingKun

0909-更新企业同步接口

上级 a4ae24a4
...@@ -20,7 +20,7 @@ def close11(cnx_,cursor_): ...@@ -20,7 +20,7 @@ def close11(cnx_,cursor_):
cursor_.close() cursor_.close()
# # 连接到Redis # # 连接到Redis
# r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) # r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
# #
# cnx = pymysql.connect(host='1.95.78.131', user='caiji', password='zzsn9988', db='caiji', # cnx = pymysql.connect(host='1.95.78.131', user='caiji', password='zzsn9988', db='caiji',
# charset='utf8mb4') # charset='utf8mb4')
......
from apscheduler.schedulers.blocking import BlockingScheduler from apscheduler.schedulers.blocking import BlockingScheduler
...@@ -29,7 +29,7 @@ def putCom(): ...@@ -29,7 +29,7 @@ def putCom():
df = pd.read_excel('D:\\企业数据\\数据组提供\\国内企业.xlsx') df = pd.read_excel('D:\\企业数据\\数据组提供\\国内企业.xlsx')
# 连接到Redis数据库 # 连接到Redis数据库
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
for i in range(len(df)): for i in range(len(df)):
social_code = df['social_code'][i] social_code = df['social_code'][i]
com_name = df['name'][i] com_name = df['name'][i]
......
...@@ -251,7 +251,7 @@ class BaseCore: ...@@ -251,7 +251,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -28,7 +28,7 @@ def check_code(com_code): ...@@ -28,7 +28,7 @@ def check_code(com_code):
return True return True
def check_date(com_code,info_date): def check_date(com_code,info_date):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
res = r.sismember('com_xqcwsj_code::'+com_code, info_date) # 注意是 保存set的方式 res = r.sismember('com_xqcwsj_code::'+com_code, info_date) # 注意是 保存set的方式
if res: if res:
return True return True
...@@ -37,7 +37,7 @@ def check_date(com_code,info_date): ...@@ -37,7 +37,7 @@ def check_date(com_code,info_date):
# 将采集后的股票代码对应的报告期保存进redis # 将采集后的股票代码对应的报告期保存进redis
def add_date(com_code,report_date): def add_date(com_code,report_date):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
res = r.sadd('com_xqcwsj_code::'+com_code,report_date) res = r.sadd('com_xqcwsj_code::'+com_code,report_date)
......
...@@ -26,7 +26,7 @@ lock = threading.Lock() ...@@ -26,7 +26,7 @@ lock = threading.Lock()
pathType = 'QYNotice/' pathType = 'QYNotice/'
taskType = '企业研报/东方财富网' taskType = '企业研报/东方财富网'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
class EsMethod(object): class EsMethod(object):
......
...@@ -29,7 +29,7 @@ lock = threading.Lock() ...@@ -29,7 +29,7 @@ lock = threading.Lock()
pathType = 'QYNotice/' pathType = 'QYNotice/'
taskType = '企业研报/东方财富网' taskType = '企业研报/东方财富网'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
class EsMethod(object): class EsMethod(object):
......
...@@ -24,7 +24,7 @@ sys.path.append('D:\\kkwork\\zzsn_spider\\base') ...@@ -24,7 +24,7 @@ sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import BaseCore import BaseCore
baseCore = BaseCore.BaseCore() baseCore = BaseCore.BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn') r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn')
obsClient = ObsClient( obsClient = ObsClient(
access_key_id='VEHN7D0TJ9316H8AHCAV', # 你的华为云的ak码 access_key_id='VEHN7D0TJ9316H8AHCAV', # 你的华为云的ak码
secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY', # 你的华为云的sk secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY', # 你的华为云的sk
...@@ -119,7 +119,7 @@ def add_check_url(article_url): ...@@ -119,7 +119,7 @@ def add_check_url(article_url):
# redis上传失败删除数据 # redis上传失败删除数据
def delete_url(article_url): def delete_url(article_url):
# r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn') # r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn')
res = r.srem('report_pdf_three_history', article_url) res = r.srem('report_pdf_three_history', article_url)
if res > 0: if res > 0:
return True return True
......
...@@ -21,7 +21,7 @@ pathType = 'QYNotice/' ...@@ -21,7 +21,7 @@ pathType = 'QYNotice/'
taskType = '企业研报/东方财富网' taskType = '企业研报/东方财富网'
es = Elasticsearch(['http://1.95.3.121:9200'], http_auth=('elastic', 'zzsn9988'), timeout=300) es = Elasticsearch(['http://1.95.3.121:9200'], http_auth=('elastic', 'zzsn9988'), timeout=300)
index_name = 'researchreportdata' index_name = 'researchreportdata'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
def searchATT(): def searchATT():
sql = "select id from clb_sys_attachment where type_id=4 and create_time>'2023-12-08' " sql = "select id from clb_sys_attachment where type_id=4 and create_time>'2023-12-08' "
......
...@@ -25,7 +25,7 @@ lock = threading.Lock() ...@@ -25,7 +25,7 @@ lock = threading.Lock()
pathType_ = 'QYResearchReport/' pathType_ = 'QYResearchReport/'
taskType = '企业研报/东方财富网' taskType = '企业研报/东方财富网'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
obsClient = ObsClient( obsClient = ObsClient(
access_key_id='VEHN7D0TJ9316H8AHCAV', # 你的华为云的ak码 access_key_id='VEHN7D0TJ9316H8AHCAV', # 你的华为云的ak码
......
...@@ -263,7 +263,7 @@ class BaseCore: ...@@ -263,7 +263,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -261,7 +261,7 @@ class BaseCore: ...@@ -261,7 +261,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -26,7 +26,7 @@ lock = threading.Lock() ...@@ -26,7 +26,7 @@ lock = threading.Lock()
pathType = 'QYNotice/' pathType = 'QYNotice/'
taskType = '企业研报/东方财富网' taskType = '企业研报/东方财富网'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
class EsMethod(object): class EsMethod(object):
......
...@@ -25,7 +25,7 @@ lock = threading.Lock() ...@@ -25,7 +25,7 @@ lock = threading.Lock()
pathType_ = 'QYResearchReport/' pathType_ = 'QYResearchReport/'
taskType = '企业研报/东方财富网' taskType = '企业研报/东方财富网'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
obsClient = ObsClient( obsClient = ObsClient(
access_key_id='VEHN7D0TJ9316H8AHCAV', # 你的华为云的ak码 access_key_id='VEHN7D0TJ9316H8AHCAV', # 你的华为云的ak码
......
...@@ -20,7 +20,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) ...@@ -20,7 +20,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
baseCore = BaseCore.BaseCore() baseCore = BaseCore.BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4') cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
cursor_ = cnx.cursor() cursor_ = cnx.cursor()
lock = threading.Lock() lock = threading.Lock()
......
...@@ -26,7 +26,7 @@ cursor_ = cnx_.cursor() ...@@ -26,7 +26,7 @@ cursor_ = cnx_.cursor()
lock = threading.Lock() lock = threading.Lock()
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
class EsMethod(object): class EsMethod(object):
......
...@@ -230,7 +230,7 @@ class BaseCore: ...@@ -230,7 +230,7 @@ class BaseCore:
self.cursor = self.cnx.cursor() self.cursor = self.cnx.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
def close(self): def close(self):
try: try:
......
import configparser import configparser
...@@ -80,7 +80,7 @@ class SinaUsstock(object): ...@@ -80,7 +80,7 @@ class SinaUsstock(object):
# 判断股票代码是否存在 # 判断股票代码是否存在
def check_code(self,com_code): def check_code(self,com_code):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
res = r.exists('com_sinacaiwushuju_code::'+com_code) res = r.exists('com_sinacaiwushuju_code::'+com_code)
#如果key存在 则不是第一次采集该企业, res = 1 #如果key存在 则不是第一次采集该企业, res = 1
if res: if res:
...@@ -89,7 +89,7 @@ class SinaUsstock(object): ...@@ -89,7 +89,7 @@ class SinaUsstock(object):
return True #表示是第一次采集 return True #表示是第一次采集
def check_date(self,com_code,info_date): def check_date(self,com_code,info_date):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
res = r.sismember('com_sinacaiwushuju_code::'+com_code, info_date) # 注意是 保存set的方式 res = r.sismember('com_sinacaiwushuju_code::'+com_code, info_date) # 注意是 保存set的方式
if res: if res:
return True return True
...@@ -98,7 +98,7 @@ class SinaUsstock(object): ...@@ -98,7 +98,7 @@ class SinaUsstock(object):
# 将采集后的股票代码对应的报告期保存进redis # 将采集后的股票代码对应的报告期保存进redis
def add_date(self,com_code,date_list): def add_date(self,com_code,date_list):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
#遍历date_list 放入redis #遍历date_list 放入redis
for date in date_list: for date in date_list:
res = r.sadd('com_sinacaiwushuju_code::'+com_code,date) res = r.sadd('com_sinacaiwushuju_code::'+com_code,date)
......
import redis import redis
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
# 获取所有键 # 获取所有键
# keys = r.keys('*') # keys = r.keys('*')
# # print(keys) # # print(keys)
...@@ -11,7 +11,7 @@ r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) ...@@ -11,7 +11,7 @@ r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
# r.srem(f_key,'[]') # r.srem(f_key,'[]')
# r.srem(f_key,'') # r.srem(f_key,'')
def check_date(com_code,info_date): def check_date(com_code,info_date):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
res = r.sismember('com_caiwushuju_code::'+com_code, info_date) # 注意是 保存set的方式 res = r.sismember('com_caiwushuju_code::'+com_code, info_date) # 注意是 保存set的方式
if res: if res:
return True return True
...@@ -20,7 +20,7 @@ def check_date(com_code,info_date): ...@@ -20,7 +20,7 @@ def check_date(com_code,info_date):
def add_date(): def add_date():
date_list = ['2023-06-30'] date_list = ['2023-06-30']
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
#遍历date_list 放入redis #遍历date_list 放入redis
for date in date_list: for date in date_list:
res = r.sadd('com_caiwushuju_code::'+'123456',date) res = r.sadd('com_caiwushuju_code::'+'123456',date)
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
import redis import redis
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
f_key = 'com_caiwushuju_code::bj851834' f_key = 'com_caiwushuju_code::bj851834'
r.set(f_key,3) r.set(f_key,3)
res = r.exists(f_key) res = r.exists(f_key)
......
import pymysql import pymysql
...@@ -23,12 +23,12 @@ cursor= cnx.cursor() ...@@ -23,12 +23,12 @@ cursor= cnx.cursor()
# 将采集后的股票代码对应的报告期保存进redis # 将采集后的股票代码对应的报告期保存进redis
# def add_date(com_code,i): # def add_date(com_code,i):
# r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3) # r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=3)
# # json_cwsj = json.dumps(dic_cwsj) # # json_cwsj = json.dumps(dic_cwsj)
# res = r.sadd('com_jingyingfenxi_code::'+com_code, i) # 注意是 保存set的方式 # res = r.sadd('com_jingyingfenxi_code::'+com_code, i) # 注意是 保存set的方式
# #
# def get_i(): # def get_i():
# r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) # r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
# i = r.get('com_jingyingfenxi_code::'+com_code) # i = r.get('com_jingyingfenxi_code::'+com_code)
def operate_analysis(i,com_code,social_code): def operate_analysis(i,com_code,social_code):
......
...@@ -17,7 +17,7 @@ sys.path.append('D:\\kkwork\\zzsn_spider\\base') ...@@ -17,7 +17,7 @@ sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import BaseCore import BaseCore
baseCore = BaseCore.BaseCore() baseCore = BaseCore.BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=5) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=5)
# cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4') # cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
cnx = baseCore.cnx_ cnx = baseCore.cnx_
obsClient = ObsClient( obsClient = ObsClient(
......
...@@ -28,7 +28,7 @@ taskType = '财务数据/纳斯达克' ...@@ -28,7 +28,7 @@ taskType = '财务数据/纳斯达克'
# 判断股票代码是否存在 # 判断股票代码是否存在
@retry(tries=3, delay=1) @retry(tries=3, delay=1)
def check_code(com_code): def check_code(com_code):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
res = r.exists('caiwu_nasdaq_code::' + com_code) res = r.exists('caiwu_nasdaq_code::' + com_code)
# 如果key存在 则不是第一次采集该企业, res = 1 # 如果key存在 则不是第一次采集该企业, res = 1
if res: if res:
...@@ -40,7 +40,7 @@ def check_code(com_code): ...@@ -40,7 +40,7 @@ def check_code(com_code):
# 判断采集日期是否存在 # 判断采集日期是否存在
@retry(tries=3, delay=1) @retry(tries=3, delay=1)
def check_date(com_code, info_date): def check_date(com_code, info_date):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
res = r.sismember('caiwu_nasdaq_code::' + com_code, info_date) # 注意是 保存set的方式 res = r.sismember('caiwu_nasdaq_code::' + com_code, info_date) # 注意是 保存set的方式
if res: if res:
return True return True
...@@ -51,7 +51,7 @@ def check_date(com_code, info_date): ...@@ -51,7 +51,7 @@ def check_date(com_code, info_date):
# 将采集后的股票代码对应的报告期保存进redis # 将采集后的股票代码对应的报告期保存进redis
@retry(tries=3, delay=1) @retry(tries=3, delay=1)
def add_date(com_code, date_list): def add_date(com_code, date_list):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=3)
# 遍历date_list 放入redis # 遍历date_list 放入redis
for date in date_list: for date in date_list:
res = r.sadd('caiwu_nasdaq_code::' + com_code, date) res = r.sadd('caiwu_nasdaq_code::' + com_code, date)
......
...@@ -28,7 +28,7 @@ baseCore = BaseCore() ...@@ -28,7 +28,7 @@ baseCore = BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
cnx = baseCore.cnx cnx = baseCore.cnx
cursor = baseCore.cursor cursor = baseCore.cursor
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=0) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=0)
import urllib3 import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
taskType = '企业负面新闻' taskType = '企业负面新闻'
......
...@@ -251,7 +251,7 @@ class BaseCore: ...@@ -251,7 +251,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -246,7 +246,7 @@ class BaseCore: ...@@ -246,7 +246,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -246,7 +246,7 @@ class BaseCore: ...@@ -246,7 +246,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -251,7 +251,7 @@ class BaseCore: ...@@ -251,7 +251,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -251,7 +251,7 @@ class BaseCore: ...@@ -251,7 +251,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -252,7 +252,7 @@ class BaseCore: ...@@ -252,7 +252,7 @@ class BaseCore:
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -21,7 +21,7 @@ cursor_ = cnx_.cursor() ...@@ -21,7 +21,7 @@ cursor_ = cnx_.cursor()
lock = threading.Lock() lock = threading.Lock()
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6) pool = redis.ConnectionPool(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
class EsMethod(object): class EsMethod(object):
def __init__(self): def __init__(self):
......
...@@ -251,7 +251,7 @@ class BaseCore: ...@@ -251,7 +251,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -251,7 +251,7 @@ class BaseCore: ...@@ -251,7 +251,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -221,7 +221,7 @@ class BaseCore: ...@@ -221,7 +221,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.__cursor_proxy= self.__cnx_proxy.cursor() self.__cursor_proxy= self.__cnx_proxy.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
# 计算耗时 # 计算耗时
def getTimeCost(self,start, end): def getTimeCost(self,start, end):
......
...@@ -263,7 +263,7 @@ class BaseCore: ...@@ -263,7 +263,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -5,7 +5,7 @@ baseCore = BaseCore() ...@@ -5,7 +5,7 @@ baseCore = BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
cnx = baseCore.cnx cnx = baseCore.cnx
cursor = baseCore.cursor cursor = baseCore.cursor
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn') r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn')
#获取所有键 #获取所有键
keys = r.keys('*') keys = r.keys('*')
......
...@@ -26,7 +26,7 @@ r = baseCore.r ...@@ -26,7 +26,7 @@ r = baseCore.r
urllib3.disable_warnings() urllib3.disable_warnings()
def check_url(sid, article_url): def check_url(sid, article_url):
# r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn') # r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn')
res = r.sismember(f'wx_url_{sid}',article_url) res = r.sismember(f'wx_url_{sid}',article_url)
if res == True: if res == True:
return True return True
...@@ -34,7 +34,7 @@ def check_url(sid, article_url): ...@@ -34,7 +34,7 @@ def check_url(sid, article_url):
return False return False
def add_url(sid, article_url): def add_url(sid, article_url):
# r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn') # r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn')
res = r.sadd(f'wx_url_{sid}', article_url, 3) # 注意是 保存set的方式 res = r.sadd(f'wx_url_{sid}', article_url, 3) # 注意是 保存set的方式
if res == 0: # 若返回0,说明插入不成功,表示有重复 if res == 0: # 若返回0,说明插入不成功,表示有重复
return True return True
......
...@@ -47,7 +47,7 @@ from bs4 import BeautifulSoup ...@@ -47,7 +47,7 @@ from bs4 import BeautifulSoup
def check_url(): def check_url():
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn',db=6)
res = r.exists('WeiXinGZH:infoSourceCode','IN-20220418-0001') res = r.exists('WeiXinGZH:infoSourceCode','IN-20220418-0001')
print(res) print(res)
if res == 1: if res == 1:
......
...@@ -262,7 +262,7 @@ class BaseCore: ...@@ -262,7 +262,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
...@@ -192,7 +192,7 @@ if __name__ == '__main__': ...@@ -192,7 +192,7 @@ if __name__ == '__main__':
# try: # try:
# googleTaskJob.r.ping() # googleTaskJob.r.ping()
# except: # except:
# googleTaskJob.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) # googleTaskJob.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
# all_keys = 'GOOGLE_KEYWORDS:COMPANY_NAME*' # all_keys = 'GOOGLE_KEYWORDS:COMPANY_NAME*'
# keys = googleTaskJob.r.scan_iter(f"{key}*") # keys = googleTaskJob.r.scan_iter(f"{key}*")
# for key in keys: # for key in keys:
......
...@@ -475,7 +475,7 @@ class BaseCore: ...@@ -475,7 +475,7 @@ class BaseCore:
try: try:
self.r.ping() self.r.ping()
except: except:
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
item = self.r.lpop(key) item = self.r.lpop(key)
return item.decode() if item else None return item.decode() if item else None
...@@ -705,7 +705,7 @@ class BaseCore: ...@@ -705,7 +705,7 @@ class BaseCore:
try: try:
self.r.ping() self.r.ping()
except: except:
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.r.rpush(key, item) self.r.rpush(key, item)
# 增加计数器的值并返回增加后的值 # 增加计数器的值并返回增加后的值
......
...@@ -13,7 +13,7 @@ from base.BaseCore import BaseCore ...@@ -13,7 +13,7 @@ from base.BaseCore import BaseCore
baseCore = BaseCore() baseCore = BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=0) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=0)
def sendKafka(dic_news): def sendKafka(dic_news):
try: try:
producer = KafkaProducer(bootstrap_servers=['1.95.3.121:9092'],max_request_size=1024*1024*20) producer = KafkaProducer(bootstrap_servers=['1.95.3.121:9092'],max_request_size=1024*1024*20)
......
...@@ -17,7 +17,7 @@ from selenium.webdriver.support import expected_conditions as EC ...@@ -17,7 +17,7 @@ from selenium.webdriver.support import expected_conditions as EC
urllib3.disable_warnings() urllib3.disable_warnings()
baseCore = BaseCore.BaseCore() baseCore = BaseCore.BaseCore()
log = baseCore.getLogger() log = baseCore.getLogger()
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
headers = { headers = {
'Accept': '*/*', 'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br', 'Accept-Encoding': 'gzip, deflate, br',
......
...@@ -2,7 +2,7 @@ import csv ...@@ -2,7 +2,7 @@ import csv
import redis import redis
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
if __name__ == "__main__": if __name__ == "__main__":
with open('./output0220_1.csv', 'w', newline='', encoding='utf-8') as file: with open('./output0220_1.csv', 'w', newline='', encoding='utf-8') as file:
writer = csv.writer(file) writer = csv.writer(file)
......
...@@ -246,7 +246,7 @@ class BaseCore: ...@@ -246,7 +246,7 @@ class BaseCore:
charset='utf8mb4') charset='utf8mb4')
self.cursor_ = self.cnx_.cursor() self.cursor_ = self.cnx_.cursor()
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.pool_caiji = PooledDB( self.pool_caiji = PooledDB(
creator=pymysql, creator=pymysql,
......
""" """
...@@ -77,7 +77,7 @@ class EsMethod(object): ...@@ -77,7 +77,7 @@ class EsMethod(object):
if __name__ == '__main__': if __name__ == '__main__':
es_method = EsMethod() es_method = EsMethod()
# 连接Redis # 连接Redis
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
result = es_method.queryatt('researchreportdata') result = es_method.queryatt('researchreportdata')
total = result['hits']['total']['value'] total = result['hits']['total']['value']
......
""" """
...@@ -53,7 +53,7 @@ class EsMethod(object): ...@@ -53,7 +53,7 @@ class EsMethod(object):
if __name__ == '__main__': if __name__ == '__main__':
es_method = EsMethod() es_method = EsMethod()
# 连接Redis # 连接Redis
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
for i in range(56): for i in range(56):
result = es_method.queryatt('subjectdatabase', i*200) result = es_method.queryatt('subjectdatabase', i*200)
total = result['hits']['total']['value'] total = result['hits']['total']['value']
......
import time import time
...@@ -35,7 +35,7 @@ def updateaunn(index_name,id,u_title): ...@@ -35,7 +35,7 @@ def updateaunn(index_name,id,u_title):
,body=body) ,body=body)
print('更新结果:%s' % result) print('更新结果:%s' % result)
if __name__ == '__main__': if __name__ == '__main__':
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
headers = { headers = {
'Proxy-Connection': 'keep-alive', 'Proxy-Connection': 'keep-alive',
'Accept': '*/*', 'Accept': '*/*',
......
import csv import csv
...@@ -2,7 +2,7 @@ import csv ...@@ -2,7 +2,7 @@ import csv
import redis import redis
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
if __name__ == "__main__": if __name__ == "__main__":
with open('./title1.csv', 'w', newline='', encoding='utf-8') as file: with open('./title1.csv', 'w', newline='', encoding='utf-8') as file:
writer = csv.writer(file) writer = csv.writer(file)
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
...@@ -207,7 +207,7 @@ def redicPullData(key, r): ...@@ -207,7 +207,7 @@ def redicPullData(key, r):
try: try:
r.ping() r.ping()
except: except:
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
item = r.lpop(key) item = r.lpop(key)
return item.decode() if item else None return item.decode() if item else None
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
...@@ -217,7 +217,7 @@ class BaseCore: ...@@ -217,7 +217,7 @@ class BaseCore:
pass pass
def __init__(self): def __init__(self):
# 连接到Redis # 连接到Redis
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
self.__cnx_proxy = pymysql.connect(host='1.95.78.131', user='caiji', password='zzsn9988', db='clb_project', self.__cnx_proxy = pymysql.connect(host='1.95.78.131', user='caiji', password='zzsn9988', db='clb_project',
charset='utf8mb4') charset='utf8mb4')
self.__cursor_proxy= self.__cnx_proxy.cursor() self.__cursor_proxy= self.__cnx_proxy.cursor()
...@@ -296,6 +296,6 @@ class BaseCore: ...@@ -296,6 +296,6 @@ class BaseCore:
try: try:
self.r.ping() self.r.ping()
except: except:
self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6) self.r = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
item = self.r.lpop(key) item = self.r.lpop(key)
return item.decode() if item else None return item.decode() if item else None
\ No newline at end of file
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论