1.run_py.sh shell后台运行文件
#!/bin/bash #name1=`basename ${0%\.*}` # 获取传入的文件名 去掉后缀 /b/c/a.py-->a #echo $name1 #name2=$(echo $name1 | sed 's/hbase_id//') # 字符串替换 #echo $name2 #NAME=$name1'.py' # $1 运行时输入参数 为文件名称 # 拼接字符串 #NAME='jmtional_browser_policy.py' # $1 运行时输入参数 为文件名称 NAME=$1 # $1 运行时输入参数 为文件名称 echo $NAME NAME=`basename ${NAME%%.*}` # 去掉文件后缀 if [ -z "$NAME" ]; then echo "STRING is empty" NAME="aa" fi echo $NAME echo "---------------" ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'` echo $ID echo "---------------" for id in $ID do kill -9 $id echo "killed $id" done echo "---------------" sleep 1 echo "chrome" echo "---------------" ID=`ps -ef | grep "chrome" | grep -v "$0" | grep -v "grep" | awk '{print $2}'` echo $ID echo "---------------" for id in $ID do kill -9 $id echo "killed $id" done echo "---------------" sleep 1 cd /home/yscredit/yoyo/sx_zc # 进入目录 sleep 1 current_dir=$(cd $(dirname $0); pwd) echo $current_dir if [ ! -d "$current_dir/logs" ]; then echo "$current_dir/logs does not exist" `mkdir $current_dir/logs` fi echo "---------------" echo "nohup python3.10 $current_dir/$NAME.py > $current_dir/logs/$NAME.log 2>&1 &" echo "---------------" echo "tail -f $current_dir/logs/$NAME.log" sleep 1 cd /home/yscredit/yoyo/sx_zc `nohup python3.10 $current_dir/$NAME.py > $current_dir/logs/$NAME.log 2>&1 &` echo "启动成功"
2.run_py.py python运行目录下的Python文件
# -*- coding: UTF-8 -*- import concurrent.futures import os import platform import re import subprocess Logger.config(level="info", processname=os.path.splitext(os.path.basename(__file__))[0]) logger = Logger # def run_main(): # system = platform.system() # path = os.path.abspath('.') # for spider_name in os.listdir("."): # if spider_name.endswith(".py") and "ggzyjy" in spider_name: # if 'indo' in system: # spider_path = '{}\{}'.format(path, spider_name) # else: # spider_path = '{}/{}'.format(path, spider_name) # logger.info(spider_path) # res = subprocess.call("python {}".format(spider_path), shell=True) # logger.info(res) def run_spider(spider_path): logger.info(spider_path) try: if os.environ.get("yu_localhosts", "") == "localhost": res = subprocess.call("python {}".format(spider_path), shell=True) else: res = subprocess.call("python3.6 {}".format(spider_path), shell=True) logger.info(res) except Exception as e: logger.error("爬虫启动失败:{]".format(e)) if __name__ == '__main__': # run_main() for i in range(1): try: data_list = [] system = platform.system() path = os.path.abspath('.') for spider_name in os.listdir("."): if spider_name.endswith(".py") and "ggzyjy" in spider_name: index = re.findall(r'\d+', spider_name) if index: index = int(index[0]) else: index = 0 if index != 14: if 'indo' in system: spider_path = '{}\{}'.format(path, spider_name) else: spider_path = '{}/{}'.format(path, spider_name) data_list.append(spider_path) with concurrent.futures.ThreadPoolExecutor(max_workers=59) as executor: future_to_url = {executor.submit(run_spider, data): data for data in data_list} except Exception as e: print("报错")
3.手动多线程运行
import subprocess class copyFile(object): def __init__(self): self.py_file = "zc_file_html00.py" self.shell_file = "hbase_id00.sh" self.start = 13 self.end = 99 # self.copy_file() self.crontab_shell() # self.run_shell() def copy_file(self): for i in range(self.start, self.end + 1): for file_name in [self.py_file, self.shell_file]: with open(file_name, 'r') as source_file: py_content = source_file.read() if i < 10: file_name_ = file_name.replace("00", "0{}".format(str(i))) else: file_name_ = file_name.replace("00", "{}".format(str(i))) with open(file_name_, 'w') as destination_file: destination_file.write(py_content) print('文件复制成功:{}'.format(file_name_)) def crontab_shell(self): for i in range(self.start, self.end + 1): b = i if b > 59: b = b - 60 if i < 10: print('{} * * * * /bin/sh /home/yscredit/yoyo/zc/hbase_id0{}.sh > /home/yscredit/yoyo/zc/hbase_id.log 2>&1 &'.format(str(b), str(i))) else: print('{} * * * * /bin/sh /home/yscredit/yoyo/zc/hbase_id{}.sh > /home/yscredit/yoyo/zc/hbase_id.log 2>&1 &'.format(str(b), str(i))) def run_shell(self): for i in range(self.start, self.end + 1): if i < 10: print(subprocess.call("bash /home/yscredit/yoyo/zc/hbase_id0{}.sh".format(str(i)), shell=True)) else: print(subprocess.call("bash /home/yscredit/yoyo/zc/hbase_id{}.sh".format(str(i)), shell=True)) if __name__ == "__main__": copyFile()
4.kills.sh 关闭某个线程
#!/bin/sh NAME=$1 # $1 运行时输入参数 为文件名称 if [ -z "$NAME" ]; then echo "STRING is empty" NAME="aa" fi echo $NAME ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'` echo $ID echo "---------------" for id in $ID do kill -9 $id echo "killed $id" done echo "---------------"
5.定时启动任务
04 18 * * * /usr/bin/bash /home/yscredit/yoyo/sx_zc/jmtional_browser_policy.sh > /home/yscredit/yoyo/sx_zc/logs/jmtional_browser_policy.log 2>&1 &