1.run_py.sh  shell后台运行文件

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/bin/bash
 
#name1=`basename ${0%\.*}`   # 获取传入的文件名  去掉后缀   /b/c/a.py-->a
#echo $name1
#name2=$(echo $name1 | sed 's/hbase_id//')   # 字符串替换
#echo $name2
#NAME=$name1'.py'  # $1 运行时输入参数  为文件名称  # 拼接字符串
#NAME='jmtional_browser_policy.py'  # $1 运行时输入参数  为文件名称
 
NAME=$1  # $1 运行时输入参数  为文件名称
echo $NAME
NAME=`basename ${NAME%%.*}`   # 去掉文件后缀
if [ -z "$NAME" ]; then
 echo "STRING is empty"
 NAME="aa"
fi
echo $NAME
echo "---------------"
ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
echo $ID
echo "---------------"
for id in $ID
do
kill -9 $id
echo "killed $id"
done
echo "---------------"
sleep 1
echo "chrome"
echo "---------------"
ID=`ps -ef | grep "chrome" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
echo $ID
echo "---------------"
for id in $ID
do
kill -9 $id
echo "killed $id"
done
echo "---------------"
 
sleep 1
cd /home/yscredit/yoyo/sx_zc   # 进入目录
 
sleep 1
 
current_dir=$(cd $(dirname $0); pwd)
echo $current_dir
if [ ! -d "$current_dir/logs" ]; then
    echo "$current_dir/logs does not exist"
    `mkdir $current_dir/logs`
fi
 
echo "---------------"
echo "nohup python3.10 $current_dir/$NAME.py > $current_dir/logs/$NAME.log 2>&1 &"
echo "---------------"
echo "tail -f $current_dir/logs/$NAME.log"
 
sleep 1
cd /home/yscredit/yoyo/sx_zc
 
`nohup python3.10 $current_dir/$NAME.py > $current_dir/logs/$NAME.log 2>&1 &`
 
echo "启动成功"

2.run_py.py  python运行目录下的Python文件

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# -*- coding: UTF-8 -*-
import concurrent.futures
import os
import platform
import re
import subprocess
 
 
Logger.config(level="info", processname=os.path.splitext(os.path.basename(__file__))[0])
logger = Logger
 
 
# def run_main():
#     system = platform.system()
#     path = os.path.abspath('.')
#     for spider_name in os.listdir("."):
#         if spider_name.endswith(".py") and "ggzyjy" in spider_name:
#             if 'indo' in system:
#                 spider_path = '{}\{}'.format(path, spider_name)
#             else:
#                 spider_path = '{}/{}'.format(path, spider_name)
#             logger.info(spider_path)
#             res = subprocess.call("python {}".format(spider_path), shell=True)
#             logger.info(res)
 
def run_spider(spider_path):
    logger.info(spider_path)
    try:
        if os.environ.get("yu_localhosts", "") == "localhost":
            res = subprocess.call("python {}".format(spider_path), shell=True)
        else:
            res = subprocess.call("python3.6 {}".format(spider_path), shell=True)
        logger.info(res)
    except Exception as e:
        logger.error("爬虫启动失败:{]".format(e))
 
 
if __name__ == '__main__':
    # run_main()
    for i in range(1):
        try:
            data_list = []
            system = platform.system()
            path = os.path.abspath('.')
            for spider_name in os.listdir("."):
                if spider_name.endswith(".py") and "ggzyjy" in spider_name:
                    index = re.findall(r'\d+', spider_name)
                    if index:
                        index = int(index[0])
                    else:
                        index = 0
                    if index != 14:
                        if 'indo' in system:
                            spider_path = '{}\{}'.format(path, spider_name)
                        else:
                            spider_path = '{}/{}'.format(path, spider_name)
                        data_list.append(spider_path)
            with concurrent.futures.ThreadPoolExecutor(max_workers=59) as executor:
                future_to_url = {executor.submit(run_spider, data): data for data in data_list}
        except Exception as e:
            print("报错")

3.手动多线程运行

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import subprocess
 
 
class copyFile(object):
    def __init__(self):
        self.py_file = "zc_file_html00.py"
        self.shell_file = "hbase_id00.sh"
        self.start = 13
        self.end = 99
        # self.copy_file()
        self.crontab_shell()
        # self.run_shell()
 
    def copy_file(self):
        for i in range(self.start, self.end + 1):
            for file_name in [self.py_file, self.shell_file]:
                with open(file_name, 'r') as source_file:
                    py_content = source_file.read()
                if i < 10:
                    file_name_ = file_name.replace("00", "0{}".format(str(i)))
                else:
                    file_name_ = file_name.replace("00", "{}".format(str(i)))
                with open(file_name_, 'w') as destination_file:
                    destination_file.write(py_content)
                print('文件复制成功:{}'.format(file_name_))
 
    def crontab_shell(self):
        for i in range(self.start, self.end + 1):
            b = i
            if b > 59:
                b = b - 60
            if i < 10:
                print('{} * * * * /bin/sh /home/yscredit/yoyo/zc/hbase_id0{}.sh > /home/yscredit/yoyo/zc/hbase_id.log 2>&1 &'.format(str(b), str(i)))
            else:
                print('{} * * * * /bin/sh /home/yscredit/yoyo/zc/hbase_id{}.sh > /home/yscredit/yoyo/zc/hbase_id.log 2>&1 &'.format(str(b), str(i)))
 
    def run_shell(self):
        for i in range(self.start, self.end + 1):
            if i < 10:
                print(subprocess.call("bash /home/yscredit/yoyo/zc/hbase_id0{}.sh".format(str(i)), shell=True))
            else:
                print(subprocess.call("bash /home/yscredit/yoyo/zc/hbase_id{}.sh".format(str(i)), shell=True))
                 
    def run_shell(self):
        for filename in os.listdir("."):
            if ".py" in filename and filename.count('_') == 2:
                print(filename)
                print(subprocess.call("bash run_py.sh {}".format(str(filename)), shell=True))
 
 
if __name__ == "__main__":
    copyFile()

4.kills.sh 关闭某个线程

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#!/bin/sh
 
NAME=$1   # $1 运行时输入参数  为文件名称
if [ -z "$NAME" ]; then
 echo "STRING is empty"
 NAME="aa"
fi
echo $NAME
ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
echo $ID
echo "---------------"
for id in $ID
do
kill -9 $id
echo "killed $id"
done
echo "---------------"

5.定时启动任务

?
1
04 18 * * * /usr/bin/bash /home/yscredit/yoyo/sx_zc/jmtional_browser_policy.sh > /home/yscredit/yoyo/sx_zc/logs/jmtional_browser_policy.log 2>&1 &