[Advanced Python] Python3 Advanced Tutorials

Ref: Python3 Advanced Tutorials，视频系列，若干问题。

Ref: Advanced-Python-Tutorial，github教程，作为补充。

 

一、Template String 

Template 属于 string 中的一个类。

from string import Template

from string import Template

class MyTemplate(Template):
  delimiter = '#'

def Main():
  cart = []
  cart.append(dict(item="Coke", price=8, qty=2))
  cart.append(dict(item="Cake", price=12, qty=1))
  cart.append(dict(item="Fish", price=32, qty=4))
  
  # t = Template("$qty x $item = $price")
  t = Template("#qty x #item = #price")
  total = 0
  print("Cart:")
  for data in cart:
    #
    # why we must use it?
    #
    print(t.substitute(data))   
    print("price is {}".format(data['price']))
    total += data["price"]   

  print('\n')
  print("Total: " + str(total))


if __name__ == '__main__':
  Main()

 

 

二、argparse

import argparse

def fib(n):
    a, b = 0, 1
    for i in range(n):
        a, b = b, a+b
    return a

def Main():
    parser = argparse.ArgumentParser()
    parser.add_argument("num", help="The input is ", type=int)
    parser.add_argument("x",   help="The input is ", type=int)
    parser.add_argument("y",   help="The input is ", type=int)

    # var num = ?
    args = parser.parse_args()
    result = fib(args.num)

    print("The " + str(args.num) +"th fib number is " + str(result))
    print("x = {}".format(args.x))
    print("y = {}".format(args.y))

if __name__ == '__main__':
    Main()

GCP的一个脚本例子：

#!/usr/bin/env python
# -*- coding:utf-8 -*-

import os
import sys
import logging
from datetime import datetime
from argparse import ArgumentParser

g_bucket = "gs://tfobd_2020_bucket"


def load_template_file():
    script_dir = os.path.dirname(os.path.abspath(__file__))
    temp_file = os.path.join(script_dir, '..', 'template', 'pipeline.config.template')
    with open(temp_file) as f:
        template = f.read()
    return template


def show_logging(level=logging.INFO):
    logger = logging.getLogger()
    h = logging.StreamHandler(stream=sys.stderr)
    h.setFormatter(
        logging.Formatter(
            fmt="%(asctime)s-line:%(lineno)d-%(levelname)s-%(message)s"
        )
    )
    logger.addHandler(h)
    logger.setLevel(level)


def check_options():
    global g_bucket

    if options.gcp_bucket:
        g_bucket = "gs://{}".format(options.gcp_bucket)

    if options.exp_name is None:
        date_time = datetime.now().strftime("%Y%m%d_%H%M%S")
        options.exp_name = "laava_{}".format(date_time)

    if options.data_dir is None:
        options.data_dir = "{}/{}_data".format(g_bucket, options.exp_name)
    else:
        options.data_dir = "{}/{}".format(g_bucket, options.data_dir)

    if options.train_dir is None:
        options.train_dir = "{}/{}_train".format(g_bucket, options.exp_name)
    else:
        options.train_dir = "{}/{}".format(g_bucket, options.train_dir)

    if options.checkpoint_file is None:
        options.checkpoint_file = "{}/{}".format(options.data_dir, 'model.ckpt')
    else:
        options.checkpoint_file = "{}/{}".format(g_bucket, options.checkpoint_file)

    if options.train_input_path is None:
        options.train_input_path = "{}/{}".format(options.data_dir, "train.record")
    else:
        options.train_input_path = "{}/{}".format(g_bucket, options.train_input_path)

    if options.train_label_map_path is None:
        options.train_label_map_path = "{}/{}".format(options.data_dir, "object-detection.pbtxt")
    else:
        options.train_label_map_path = "{}/{}".format(g_bucket, options.train_label_map_path)

    if options.test_input_path is None:
        options.test_input_path = "{}/{}".format(options.data_dir, "test.record")
    else:
        options.test_input_path = "{}/{}".format(g_bucket, options.test_input_path)

    if options.test_label_map_path is None:
        options.test_label_map_path = "{}/{}".format(options.data_dir, "object-detection.pbtxt")
    else:
        options.test_label_map_path = "{}/{}".format(g_bucket, options.test_label_map_path)


def generate_config_file(template):
    new_config = template
    new_config = new_config.replace('{num_classes}', str(options.num_classes))
    new_config = new_config.replace('{fine_tune_checkpoint}', options.checkpoint_file)
    new_config = new_config.replace('{num_steps}', str(options.total_steps))
    new_config = new_config.replace('{train_input_path}', options.train_input_path)
    new_config = new_config.replace('{train_label_map_path}', options.train_label_map_path)
    new_config = new_config.replace('{eval_input_path}', options.test_input_path)
    new_config = new_config.replace('{eval_label_map_path}', options.test_label_map_path)
    new_config = new_config.replace('{delay}', str(options.delay_steps))
    return new_config


def make_exp_dir(exp_name):
    cur_dir = os.path.dirname(os.path.abspath(__file__))
    exp_dir = os.path.join(cur_dir, '..', 'experiment', exp_name)
    os.makedirs(exp_dir, exist_ok=True)
    return exp_dir


def fsave_config(config, fpath):
    fpath = fpath if fpath else 'pipeline.config'
    with open(fpath, 'w') as f:
        f.write(config)


def main(options, arguments):
    show_logging()
    check_options()
    exp_dir = make_exp_dir(options.exp_name)
    template = load_template_file()
    new_config = generate_config_file(template)
    fpath = os.path.join(exp_dir, 'pipeline.config')
    fsave_config(new_config, fpath)



if __name__ == "__main__":

    option_0 = {'name': ('-n', '--exp-name'), 'nargs': '?',
                'help': '实验名称'}
    option_1 = {'name': ('-d', '--data-dir'), 'nargs': '?',
                'help': 'GCP上实验数据所在目录'}
    option_2 = {'name': ('-t', '--train-dir'), 'nargs': '?',
                'help': 'GCP上训练结果所在目录（相对路径）'}
    option_3 = {'name': ('-c', '--num-classes'), 'nargs': '?', 'default': 10,
                'help': '分类类别'}
    option_4 = {'name': ('-1', '--checkpoint-file'), 'nargs': '?',
                'help': 'GCP上预训练模型文件相对路径'}
    option_5 = {'name': ('-2', '--train-input-path'), 'nargs': '?',
                'help': 'GCP上训练数据文件相对路径'}
    option_6 = {'name': ('-3', '--train-label-map-path'), 'nargs': '?',
                'help': 'GCP上训练数据标签映射文件相对路径'}
    option_7 = {'name': ('-4', '--test-input-path'), 'nargs': '?',
                'help': 'GCP上测试数据文件相对路径'}
    option_8 = {'name': ('-5', '--test-label-map-path'), 'nargs': '?',
                'help': 'GCP上测试数据标签映射文件相对路径'}
    option_9 = {'name': ('-6', '--total-steps'), 'nargs': '?', 'type': int, 'default': 4000,
                'help': '训练总轮数'}
    option_X = {'name': ('-7', '--delay-steps'), 'nargs': '?', 'type': int, 'default': 500}
    option_g = {'name': ('-g', '--gcp-bucket'), 'nargs': '?'}

    options = [option_0, option_1, option_2, option_3,
               option_4, option_5, option_6, option_7,
               option_8, option_9, option_X, option_g]


    parser = ArgumentParser()

    for option in options:
        param = option['name']
        del option['name']
        parser.add_argument(*param, **option)

    options = parser.parse_args()
    arguments = sys.argv

    # 两个参数：解释器，参数内容
    main(options, arguments)

 

下面的 getopt 貌似更好用。

import sys
import getopt
import os
from os import listdir, mkdir
from os.path import exists, join


def Main():
    try:
        options, args = getopt.getopt(sys.argv[1:], "hi:", ["help","","input=","output="])
    except getopt.GetoptError:
        sys.exit()
    for name, value in options:
        print("{} -- {}".format(name, value))
        if name in ("-h","--help"):
            usage()
        if name in ("-i","--input"):
            print("input is {}".format(value))
            dataset_dir_path = value
        if name in ("-o","--output"):
            print("output is {}".format(value))
            output_dir_path = value

    if not exists(dataset_dir_path):
        print("Error: cannot find input folder: {}".format(dataset_dir_path))
        exit()
    else:
        print("Info: folder is {}".format(dataset_dir_path))

    if not exists(output_dir_path):
        print("Error: cannot find output folder: {}".format(output_dir_path))
        exit()
    else:
        print("Info: folder is {}".format(output_dir_path))


if __name__ == '__main__':
    Main()

执行效果：

$ python 03_02_argparse.py  --input ./in --output ./out
--input -- ./in
input is ./in
--output -- ./out
output is ./out
Error: cannot find input folder: ./in

 

 

三、Regular Expressions

简单的例子：

#coding=utf-8

import re

def Main():
    line = "I think I understand regular expressions"
    
    matchResult = re.match('think', line, re.M|re.I)
    if matchResult:
        print("Match Found: " + matchResult.group())
    else:
        print("No Match was Found.")

    searchResult = re.search('think', line, re.M|re.I)
    if searchResult:
        print("Match Found: " + searchResult.group())
    else:
        print("Nothing found in search.")

if __name__ == "__main__":
    Main()

 

 

四、MultiThreading

基本套路：threading

from threading import Thread
import time

def timer(name, delay, repeat):
    print("Timer: " + name + "Started")
    while repeat > 0:
        time.sleep(delay)
        print(name + ": " + str(time.ctime(time.time())))
        repeat -= 1
    print("Timer: " + name + "Completed")

def Main():
    # 第一步
    t1 = Thread(target=timer, args=("Timer1", 1, 5))
    t2 = Thread(target=timer, args=("Timer2", 2, 5))
    # 第二步
    t1.start()
    t2.start()

    print("Main Completed")

if __name__ == "__main__":
    Main()

进程和线程的对比。

真线程，还是得调用C库：Python创建真正的多线程

#coding=utf-8
from multiprocessing import Pool
from threading import Thread

from multiprocessing import Process


def loop():
    while True:
        pass

if __name__ == '__main__':

    for i in range(3):
        # t = Process(target=loop)  # 打开进程模式
        t = Thread(target=loop)  # 打开线程模式
        t.start()

    while True:
        pass

 

 

五、Networking

continue...