python管理云服务器

如今是云时代，公司买服务器也从传统的IDC托管到现在的各大云厂商采购 。这里，我们将以阿里云、腾讯云为例实现云服务器实例的获取。

1、首先部署django环境，然后安装django drf, 把drf注册到APPS中

INSTALLED_APPS = [
    ...
    'rest_framework',
]

2、在项目下新建一个Python Package命名为apps，settting.py配置路径， IDE把apps设置成Mark Directory as source root

import os
import sys
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(BASE_DIR, 'apps'))

3、在apps可以创建app了, 创建python package命名为resources,然后把resources注册到APPS中

4、在resources下新建个models.py设计数据表，这一步，可以先获取阿里云和腾讯云的实例返回结果，然后再进行设计。然后把表同步到数据库

from django.db import models


class Cloud(models.Model):
    name = models.CharField(max_length=50, verbose_name='云厂商名称', help_text='云厂商名称')
    code = models.CharField(max_length=20, verbose_name='云厂商名称', help_text='云厂商名称')

    def __str__(self):
        return self.name


class Server(models.Model):
    cloud = models.ForeignKey(Cloud, verbose_name='云厂商', help_text='云厂商')
    instanceId = models.CharField(max_length=50, db_index=True, verbose_name='实例ID', help_text='实例ID')
    instanceName = models.CharField(max_length=50, db_index=True, verbose_name='实例名', help_text='实例名')
    instanceType = models.CharField(max_length=50, verbose_name='实例类型', help_text='实例类型')
    osName = models.CharField(max_length=32, verbose_name='操作系统', help_text='操作系统')
    cpu = models.CharField(max_length=32, verbose_name='cpu核数', help_text='cpu核数')
    memory = models.CharField(max_length=32, verbose_name='内存G', help_text='内存G')
    createdTime = models.DateTimeField(verbose_name='创建时间',help_text='创建时间', null=True, blank=True)
    expiredTime = models.DateTimeField(verbose_name='到期时间', help_text='到期时间', null=True, blank=True)


class Ip(models.Model):
    ip = models.GenericIPAddressField(verbose_name='IP地址', help_text='IP地址', null=True, blank=True)
    inner = models.ForeignKey(Server, null=True, related_name='privateIpAddresses', verbose_name='内网IP', help_text='内网IP')
    public = models.ForeignKey(Server, null=True, related_name='publicIpAddresses', verbose_name='外网IP', help_text='外网IP')
models.py

5、在 resources新建两个python package分别命名为aliyun(阿里云)、qcloud(腾讯云), 加下来就需要用到他们的Python SDK了

6、在settings.py中配置他们的secretId、secretKey、regions(区域)

#ALIYUN
ALIYUN_SECRETID = 'xxxxxxxxxxx'
ALIYUN_SECRETKEY = 'xxxxxxxxxxxxxxxxx'
ALIYUN_REGIONS = ["xxxx"]

#QCLOUD
QCLOUD_SECRETID = 'xxxxxxxxxxx'
QCLOUD_SECRETKEY = 'xxxxxxxxxxx'
QCLOUD_REGIONS = ["xxxx"]
settings.py

阿里云

编辑aliyun.__init__.py

from django.conf import settings
from aliyunsdkcore import client


def getClient():
    for region in settings.ALIYUN_REGIONS:
        try:
            return client.AcsClient(settings.ALIYUN_SECRETID, settings.ALIYUN_SECRETKEY, region)
        except Exception as err:
            print('获取阿里云client失败：', err)
aliyun.__init__.py

新建aliyun.ecs.py

import json
from resources.aliyun import getClient
from aliyunsdkecs.request.v20140526.DescribeInstancesRequest import DescribeInstancesRequest
from resources.serializers import ServerSerializer


def getInnerIps(VpcAttributes):
    return VpcAttributes['PrivateIpAddress']['IpAddress']


def getPlublicIps(EipAddress):
    ip_list = []
    ip_list.append(EipAddress['IpAddress'])
    return ip_list


def saveInstance(instance):
    # print(instance)
    data = {}
    data['cloud'] = 'aliyun'
    data['instanceId'] = instance['InstanceId']
    data['instanceName'] = instance['InstanceName']
    data['instanceType'] = instance['InstanceType']
    data['osName'] = instance['OSName']
    data['cpu'] = instance['Cpu']
    data['memory'] = int(instance['Memory'] / 1024)
    data['createdTime'] = instance['CreationTime']
    data['expiredTime'] = instance['ExpiredTime']
    data['innerIps'] = getInnerIps(instance['VpcAttributes'])
    data['publicIps'] = getPlublicIps(instance['EipAddress'])
    serializer = ServerSerializer(data=data)
    if serializer.is_valid():
        serializer.save()
    else:
        print('阿里云序列化错误: ', serializer.errors)

def getEcsList():
    client = getClient()
    request = DescribeInstancesRequest()
    request.set_accept_format('json')
    request.set_InstanceNetworkType = 'vpc'
    request.set_PageSize(100)
    try:
        resp = client.do_action(request)
        data = json.loads(resp)
        instances = data['Instances']['Instance']
        # saveInstance(instances[0])
        print(len(instances))
        for instance in instances:
            saveInstance(instance)
    except Exception as err:
        print('获取阿里云实例失败:', err)
aliyun.ecs.py

腾讯云

编辑qcloud.__init__.py

from django.conf import settings
from tencentcloud.common import credential
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException


def getCredential():
    try:
        return credential.Credential(settings.QCLOUD_SECRETID, settings.QCLOUD_SECRETKEY)
    except TencentCloudSDKException as err:
        print(err)
qcloud.__init__.py

创建qcloud.cvm.py

import json
from django.conf import settings
from tencentcloud.cvm.v20170312 import cvm_client, models
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
from resources.qcloud import getCredential
from resources.serializers import ServerSerializer


def getCvmClient(region):
    cred = getCredential()
    try:
        return cvm_client.CvmClient(cred, region)
    except TencentCloudSDKException as err:
        print('获取腾讯云client:错误: ', err)


def saveInstance(instance):
    #print(instance)
    data = {}
    data['cloud'] = 'qcloud'
    data['instanceId'] = instance['InstanceId']
    data['instanceName'] = instance['InstanceName']
    data['instanceType'] = instance['InstanceType']
    data['osName'] = instance['OsName']
    data['cpu'] = instance['CPU']
    data['memory'] = instance['Memory']
    data['innerIps'] = instance['PrivateIpAddresses']
    data['publicIps'] = instance['PublicIpAddresses']
    data['createdTime'] = instance['CreatedTime']
    data['expiredTime'] = instance['ExpiredTime']

    serializer = ServerSerializer(data=data)
    if serializer.is_valid():
        serializer.save()
    else:
        print('腾讯云序列化错误: ', serializer.errors)

def getCvmList():
    for region in settings.QCLOUD_REGIONS:
        try:
            client = getCvmClient(region)
            req = models.DescribeInstancesRequest()
            resp = client.DescribeInstances(req)
            data = json.loads(resp.to_json_string())
            instances = data['InstanceSet']
            for instance in instances:
                saveInstance(instance)
        except Exception as err:
            print('获取腾讯云主机失败: ', err)
qcloud.cvm.py 

序列化和反序列化

创建resources.serializers.py

from rest_framework import serializers
from . models import Cloud, Server, Ip


class ServerSerializer(serializers.Serializer):
    id = serializers.ReadOnlyField()
    cloud = serializers.PrimaryKeyRelatedField(queryset=Cloud.objects.all(),help_text='云厂商', many=False)
    instanceId = serializers.CharField(required=True, help_text='实例ID')
    instanceName = serializers.CharField(required=True, help_text='实例名')
    instanceType = serializers.CharField(required=True, help_text='实例类型')
    osName = serializers.CharField(required=True, help_text='操作系统')
    cpu = serializers.CharField(required=True, help_text='CPU核数')
    memory = serializers.CharField(required=True, help_text='内存G数')
    innerIps = serializers.ListField(help_text='内网IP', write_only=True)
    publicIps = serializers.ListField(help_text='外网IP', write_only=True)
    createdTime = serializers.DateTimeField(help_text='创建时间')
    expiredTime = serializers.DateTimeField(help_text='到期时间')


    def getCloudPk(self, code):
        try:
            obj = Cloud.objects.get(code__exact=code)
            return obj.id
        except Cloud.DoesNotExist:
            print('云厂商不存在')
            raise serializers.ValidationError('云厂商不存在')


    def to_internal_value(self, data):
        data['cloud'] = self.getCloudPk(data['cloud'])
        return super(ServerSerializer, self).to_internal_value(data)


    def getInstance(self, instanceId):
        try:
            return Server.objects.get(instanceId__exact=instanceId)
        except Server.DoesNotExist:
            return None
        except Exception as err:
            raise  serializers.ValidationError(err.args)

    def create(self, validated_data):
        instance = self.getInstance(validated_data['instanceId'])
        if instance is not None:
            return self.update(instance, validated_data)
        innerIps = validated_data.pop('innerIps')
        publicIps = validated_data.pop('publicIps')
        instance = Server.objects.create(**validated_data)
        self.check_inner_ip(innerIps, instance)
        self.check_public_ip(publicIps, instance)
        return instance

    def check_inner_ip(self, innerIps, instance):
        ip_queryset = instance.privateIpAddresses.all()
        current_ip_obj = []
        for ip in innerIps:
            try:
                ip_obj = Ip.objects.get(ip__exact=ip)
            except Ip.DoesNotExist:
                ip_obj = Ip.objects.create(ip=ip, inner=instance)
            current_ip_obj.append(ip_obj)
        not_exist_ip = set(ip_queryset) - set(current_ip_obj)
        for ip_obj in not_exist_ip:
            ip_obj.delete()


    def check_public_ip(self, publicIps, instance):
        ip_queryset = instance.publicIpAddresses.all()
        current_ip_obj = []
        for ip in publicIps:
            try:
                ip_obj = Ip.objects.get(ip__exact=ip)
            except Ip.DoesNotExist:
                ip_obj = Ip.objects.create(ip=ip, public=instance)
            current_ip_obj.append(ip_obj)
        not_exist_ip = set(ip_queryset) - set(current_ip_obj)
        for ip_obj in not_exist_ip:
            ip_obj.delete()


    def update(self, instance, validated_data):
        instance.instanceName = validated_data.get('instanceName')
        instance.osName = validated_data.get('osName')
        instance.cpu = validated_data.get('cpu')
        instance.memory = validated_data.get('memory')
        instance.createdTime = validated_data.get('createdTime')
        instance.expiredTime = validated_data.get('expiredTime')
        instance.save()
        self.check_inner_ip(validated_data['innerIps'], instance)
        self.check_public_ip(validated_data['publicIps'], instance)
        return instance

    def to_representation(self, instance):
        ret = super(ServerSerializer, self).to_representation(instance)
        ret['cloud'] = {
            'id': instance.cloud.id,
            'name': instance.cloud.name
        }
        ret['innerIps'] = [ip.ip for ip in instance.privateIpAddresses.all()]
        ret['publicIps'] = [ip.ip for ip in instance.publicIpAddresses.all()]
        return ret
resources.serializers.py

新建resources.views.py

from django.http import HttpResponse
from rest_framework import viewsets
from django.views import View
from resources.qcloud import cvm
from resources.aliyun import ecs
from . serializers import ServerSerializer
from . models import Server

class Qtest(View):
    def get(self, request, *args, **kwargs):
        cvm.getCvmList()
        return HttpResponse('Qtest')

class Atest(View):
    def get(self, request, *args, **kwargs):
        ecs.getEcsList()
        return HttpResponse('Atest')


class ServerViewSet(viewsets.ModelViewSet):
    queryset = Server.objects.all()
    serializer_class = ServerSerializer
resources.views.py

新建resources.router.py

from rest_framework.routers import DefaultRouter
from . views import ServerViewSet

router = DefaultRouter()
router.register('servers', ServerViewSet, base_name='servers')
resources.router

新建resourcrs.urls.py

from django.conf.urls import url
from . views import Qtest, Atest


urlpatterns = [
    url(r'^qtest/', Qtest.as_view(), name='qtest'),
    url(r'^atest/', Atest.as_view(), name='atest'),
]
resources.urls.py

编辑根urls.py

from django.conf.urls import url, include
from rest_framework.routers import DefaultRouter
from resources.router import router as resources_router


router = DefaultRouter()
router.registry.extend(resources_router.registry)

urlpatterns = [
    url(r'^test/', include('resources.urls')),
    url(r'', include(router.urls)),
]
urls.py

采集数据

访问http://127.0.0.1:8000/test/atest 采集阿里云实例数据

访问http://127.0.0.1:8000/test/qtest采集腾讯云实例数据

 任务调度

上面我们是通过http请求来进行数据采集，这只是一个用来测试的方法，在实际应用中，我们可以使用apscheduler任务调度模块进行定时任务。

安装django-apscheduler模块

pip install django-apscheduler

注册到APPS中

INSTALLED_APPS = [
    ...
    'django_apscheduler',
]

django-apscheduler模块会有两张表，django_apscheduler_djangojob （定义的任务），django_apscheduler_djangojobexecution（记录任务执行情况包括出现的异常）

所以我们需要同步下数据库

python manage.py makemigrations django_apscheduler
python manage.py migrate apscheduler

新建resources. apscheduler,py

from datetime import datetime
from apscheduler.schedulers.background import BackgroundScheduler
from django_apscheduler.jobstores import  DjangoJobStore, register_job, register_events

#建立一个后台执行的任务
scheduler = BackgroundScheduler()

#任务添加存储
scheduler.add_jobstore(DjangoJobStore(), 'default')

#任务调度，每3秒执行一次
@register_job(scheduler, "interval", seconds=3)
def myjob():
    print('my name is heboan--{}'.format(datetime.now()))

#任务执行器
register_events(scheduler)

scheduler.start()

现在只需要在项目根urls.py导入resources. apscheduler即可

from resources import apscheduler

启动项目后，可以看到它每3秒就执行了一次，有兴趣也可以去数据库查看那两张表

我们也可以让任务在前台运行，这次我们把要执行的任务换成前面的云服务器数据采集。

先把之前根urls.py那个导入模块去掉

编辑resources. apscheduler,py

from datetime import datetime
from apscheduler.schedulers.background import BackgroundScheduler
from django_apscheduler.jobstores import  DjangoJobStore, register_job, register_events
from resources.qcloud.cvm import getCvmList
from resources.aliyun.ecs import getEcsList
from apscheduler.schedulers.blocking import BlockingScheduler


#scheduler = BackgroundScheduler()
#建立前台任务
scheduler = BlockingScheduler()
scheduler.add_jobstore(DjangoJobStore(), 'default')

@register_job(scheduler, "interval", seconds=30)
def sync_cloud():
    getEcsList() #采集阿里云
    getCvmList() # 采集腾讯云


register_events(scheduler)

在resources下新建Python Package命名为management

在management下新建Python Package命名为commands

在 commands下新建文件schedule.py

现在我们执行下python manage.py，可以看到多出来一条可以执行的命令

 现在，我们来编辑schedule.py

from django.core.management.base import BaseCommand
from resources.apscheduler import scheduler


class Command(BaseCommand):
    help = "django schedule"
    def handle(self, *args, **options):
        scheduler.start()

最后我们就可以执行python manage.py schedule了

 

前台任务：只需要保持python manage.py schedule在运行就可以了，后台任务需要项目跑起来。看你喜欢怎么用了 ^_^

分页

当我们访问http://127.0.0.1:8000/servers/ 会把所有的实例都显示出来，比较好的做法是进行分页显示，最简单的方式就在settings.py种添加如下配置：

REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
    'PAGE_SIZE': 10    #每页显示的条数
}

这样整个全局就生效了，按每页显示10条进行分页

 当然，我们也可以自定义分页，如下：

1、在项目（devops）下新建个文件paginations.py

from rest_framework.pagination import PageNumberPagination


class Pagination(PageNumberPagination):
    page_size = 10   #每页显示的条数
    page_size_query_param = 'page_size'   #想要显示的页数
    page_query_param = 'p'   #指定页码的参数
    max_page_size = 100   每页显示的最大条数

2、修改settings.py中的分页设置（这个也会导致全局生效）

REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'devops.paginations.Pagination',  #这里配置成我们自定义的
    'PAGE_SIZE': 10
}

上面的配置完成后，就会全局使用我们自定义的分页配置，如果只只想要Servers使用自定义的分页配置，那么我们就不需要修改settings.py，只需要如下操作

...
from devops.paginations import Pagination


class ServerViewSet(viewsets.ModelViewSet):
    queryset = Server.objects.all()
    serializer_class = ServerSerializer
    pagination_class = Pagination   #指定自定义的分页类即可

搜索

服务器实例比较多，如果我们想要搜索出指定的服务器实例，比如根据instanceName来搜索（http://127.0.0.1:8000/servers/?instanceName=pr-es-01）

现在我们在浏览器访问http://127.0.0.1:8000/servers/?instanceName=pr-es-01， 会发现并没有做任何过滤!

那么我们该怎么办？

　　1、我们进行搜索其实就是对模型进行条件筛选

　　2、在drf中，就是对queryset进行操作了

class ServerViewSet(viewsets.ModelViewSet):
    queryset = Server.objects.all()
    serializer_class = ServerSerializer
    pagination_class = Pagination

    def get_queryset(self):
        queryset = super(ServerViewSet, self).get_queryset()
        #获取到instanceName参数
        instanceName = self.request.query_params.get('instanceName')
        #如果有传递此参数则进行instanceName模糊查询
        if instanceName:
            queryset = queryset.filter(instanceName__icontains=instanceName)
        return queryset

但是一般我们不会使用上面的方法去实现搜索。我们会使用django-filter

当然首先必须要先安装

pip install django-filter

然后注册到APPS中

INSTALLED_APPS = [
    ...
    'django_filters',
]

然后可以在视图中使用了

...
from django_filters.rest_framework import DjangoFilterBackend


class ServerViewSet(viewsets.ModelViewSet):
    queryset = Server.objects.all()
    serializer_class = ServerSerializer
    pagination_class = Pagination

    #使用过滤器
    filter_backends = (DjangoFilterBackend,)
    filter_fields = ("instanceName",)

为了支持模糊查询或其他更高级的查询，我们就需要自定义过滤器了

新建文件resources.filters.py

from django_filters.rest_framework import FilterSet
import django_filters
from resources.models import Server


class ServerFilter(FilterSet):
    instanceName = django_filters.CharFilter(lookup_expr="icontains") #支持模糊查询
    class Meta:
        model = Server
        fields = ['instanceName']

然后在视图中指定过滤类就可以了

from resources.filters import ServerFilter

class ServerViewSet(viewsets.ModelViewSet):
    queryset = Server.objects.all()
    serializer_class = ServerSerializer
    pagination_class = Pagination
    filter_backends = (DjangoFilterBackend,)  #可以放到全局配置中
    filter_class = ServerFilter

上面我提到filter_backends = (DjangoFilterBackend,)可以放到全局配置中，这样视图中就不需要配置了。操作如下：

1、先把视图中这段配置去掉 ： filter_backends = (DjangoFilterBackend,)

2、编辑settings.py

REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'devops.paginations.Pagination',
    'PAGE_SIZE': 10,
    'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',)  #加上此配置即可
}

 

ffilter还可以进行一些高级的方法,比如可以查询多个字段匹配

import django_filters

from django.contrib.auth import get_user_model
from django.db.models import Q


User = get_user_model()

class UserFilter(django_filters.rest_framework.FilterSet):
    """
    用户过滤类
    """
    username = django_filters.CharFilter(method='search_username')

    def search_username(self, queryset, name, value):
        return queryset.filter(Q(name__icontains=value)|Q(username__icontains=value))


    class Meta:
        model = User
        fields = ['username']

 

posted @ 2018-11-30 10:45  sellsa  阅读(426)  评论(0)  编辑  收藏  举报