jenkins發布程序觸發shell調用python腳本刷新akamai cdn api

刷新cdn的流程：
jenkins獲取git中的代碼，觸發腳本推送到生產環境中(即cdn的源站) --> 觸發腳本獲取git工作目錄的更新列表，將更新列表拼湊成帶域名信息的url，寫入到目錄中 --> 觸發python腳本讀取目錄中的url發送給akamai的api進行cdn的刷新

參考文檔創建client api，此次我們的賬號沒有創建client api的權限，需要管理員處理
文檔地址：https://developer.akamai.com/api/getting-started#beforeyoubegin

創建和api交互的client api后會得到類似如下信息

client_secret = "pass"
host = "host.purge.akamaiapis.net"
access_token = "token"
client_token = "client_token"

api參考地址：

https://github.com/akamai/api-kickstart

jenkins配合shell腳本觸發python自動刷新akamai cdn

項目背景：
設計中心開啟了統一資源管理系統neveragain.chinaosft.com，即公司后續新開發的站點引用的image,css,js等公共資源統一從該系統獲取

需求：
當代碼合並到master的指定目錄后需要自動發布代碼及圖片等資源到生產環境並且能即時刷新CDN

轉化為運維需求：
git中提交代碼並且合並master --> 使用jenkins發布代碼到 neveragain.chinaosft.com 所在服務器 --> 獲取更新的代碼並且刷新CND


具體實現過程
1.配置jenkins讓jenkins能拉取代碼到jenkins服務器，因配置較為簡單，此處略

2.配置發布的腳本：

腳本的主要邏輯：發布指定代碼到生產環境服務器 --> 同時獲取代碼中dist目錄更新的文件，將文件拼湊成CDN的api可以識別的URL --> 使用python腳本讀取需要更新的URL列表並且觸發AKAMAI CDN API刷新資源

jenkins中的shell腳本

[root@jenkins:/usr/local/worksh/jeninks_task]# cat neveragain_chinasoft_com.sh 
#!/bin/bash
#############################################
# 通過jenkins發布任務 neveragain.chinasoft.com 發布  注意/data/www/vhosts/neveragain.chinasoft.com/httpdocs/dist/ 發布到線上對應的是2019目錄

cart_iplist="1.1.1.1"

function neveragain_chinasoft_eus_rsync()
{
for ip in $cart_iplist
do
        echo "-- start pub --- 預發布到外網 ${ip} ----------------------------------------"
    /usr/local/bin/rsync -vazP --bwlimit=1000 --exclude='.git/' --exclude='.gitignore/' --password-file=/data/www/.rsync/rsyncd.pass /data/www/vhosts/neveragain.chinasoft.com/httpdocs/dist/ apache@$ip::apache/data/www/vhosts/neveragain.chinasoft.com/httpdocs/2019/
    if [[ $? == 0 || $? == 23 ]];then
            rsync_edit=1
    else
            rsync_edit=0
            echo "`date` rsync發布失敗! -> editUrls.txt"
            exit 1
    fi

    echo -e "-- end pub ${ip} ----------------------------------------------------------\n\n"
done
}

# 執行同步
neveragain_chinasoft_eus_rsync

# 讀取git的更新列表，發送請求調用python腳本刷新akamai CDN
function update_cdn
{
    # 工作目錄
    WORKSPACE="/data/jenkins_home/workspace/DesignCenter.neveragain.chinasoft.com/"
    cd $WORKSPACE

    # 獲取git變更列表
    changefiles=$(git diff --name-only HEAD~ HEAD)
    #echo $changefiles
    # 先把文件路徑寫死，作為測試使用
    #changefiles="dist/assets/image/box/drfone-mac.png dist/assets/image/box/drfone-win.png dist/assets/image/box/dvdcreator-mac.png dist/assets/image/box/dvdcreator-win.png"

    #20190812103337
    now_time="`date +%Y%m%d%H%M%S`"
    # 將更新的文件列表寫入日志文件中
    for newfile in $changefiles;
    do
        start_str=${newfile:0:4}
        #echo $start_str
        # 如果變更的文件是 dist 目錄下的文件就觸發該文件刷新CDN
        if [ $start_str == 'dist' ];then
            need_file=${newfile:5}
            #echo $need_file
            need_url="https://neveragain.chinasoft.com/2019/$need_file"
            #echo $need_url
            echo "${need_url}" >> "/usr/local/worksh/jeninks_task/akamai_api/logs/${now_time}.log"
        fi
    done

# 調用Python腳本刷新cdn
/usr/local/worksh/jeninks_task/akamai_api_venv/bin/python /usr/local/worksh/jeninks_task/akamai_api/akamai_api.py $now_time
if [ $? != 0 ];then
    echo "刷新CDN失敗"
    exit 1
else
    echo "刷新CDN成功"
fi

}
# 刷新cdn
update_cdn

# python腳本

# 刷新cdn的python腳本結構
[root@jenkins:/usr/local/worksh/jeninks_task/akamai_api]# tree
.
├── akamai_api.py
├── lib
│   ├── http_calls.py
│   ├── __init__.py
├── logs
│   ├── 20190814164047.log
│   └── 20190814172256.log
├── log.txt
├── README.md
└── requirement.txt

# cat /usr/local/worksh/jeninks_task/akamai_api/logs/20190814172256.log
https://neveragain.chinasoft.com/2019/assets/icon/brand/finder.svg
https://neveragain.chinasoft.com/2019/assets/icon/logo/edraw-horizontal-white.png

# 主程序
[root@jenkins:/usr/local/worksh/jeninks_task]# cat /usr/local/worksh/jeninks_task/akamai_api/akamai_api.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @File  : akamai_api.py
# @Desc  : 讀取指定文件中url的路徑內容，刷新Akamai CDN的邊緣節點數據

import requests, json,time,os,sys
from lib.http_calls import EdgeGridHttpCaller
from akamai.edgegrid import EdgeGridAuth
import logging

class Akamai_API():
    def __init__(self,api_host,api_access_token,api_client_token,api_client_secret,verbose=False,debug=False,action="delete",network="production"):
        self.host = api_host
        self.access_token = api_access_token
        self.client_token = api_client_token
        self.client_secret = api_client_secret

        self.verbose = verbose
        self.debug = debug

        #API的清除動作:delete invalidate
        self.action =action
        self.network =network

        self.session = requests.Session()

    def __auth(self):
        self.session.auth = EdgeGridAuth(
            client_token=self.client_token,
            client_secret=self.client_secret,
            access_token=self.access_token
        )
        return self.session

    def postPurgeRequest(self,refush_url_list):
        self.__auth()
        baseurl = '%s://%s/' % ('https', self.host)
        httpCaller = EdgeGridHttpCaller(self.session, self.debug, self.verbose, baseurl)
        purge_obj = {
            # "objects": [
            #     "https://bc.akamaiapibootcamp.com/index.html"
            # ]
            "objects": refush_url_list,
        }
        # print("Adding %s request to queue - %s" % (self.action, json.dumps(purge_obj)))
        purge_post_result = httpCaller.postResult('/ccu/v3/%s/url/%s' % (self.action, self.network), json.dumps(purge_obj))
        return purge_post_result

def ReadFile(filename):
    """
    讀取文件內容中的url路徑
    每行一條url路徑
    """
    l = []
    error_l = []
    with open(filename) as f:
        for url in  f.readlines():
            url_str = str(url).strip("\n")
            if str(url).startswith("https://neveragain.chinasoft.com"):
                l.append(url_str)
            else:
                error_l.append(url_str)
    if error_l:
        raise Exception("The format of the file path is incorrect. %s"%('\n'.join(error_l)))
    return l

if __name__ == "__main__":
    #API信息
    API_HOST = "host.purge.akamaiapis.net"
    API_ACCESS_TOKEN = "token"
    API_CLIENT_TOKEN = "client_token"
    API_CLIENT_SECRET = "api_client_secret="

    api = Akamai_API(api_host=API_HOST,api_access_token=API_ACCESS_TOKEN,api_client_token=API_CLIENT_TOKEN,api_client_secret=API_CLIENT_SECRET)

    #接收url文件名稱
    if len(sys.argv) != 2:
        raise Exception("Not enough parameters for %s"%sys.argv[0])
    prefix_url_filename = sys.argv[1]

    # 定義日志級別
    baseDir = os.path.dirname(os.path.abspath(__file__))
    logfile = os.path.join(baseDir,"log.txt")
    logging.basicConfig(level=logging.INFO,
                        filename=logfile,
                        filemode='a',
                        format='%(asctime)s - %(filename)s - %(levelname)s: %(message)s')
    logger = logging.getLogger(__name__)

    #讀取url的文件內容
    filename = os.path.join(baseDir,os.path.join("logs","%s.log"%prefix_url_filename))
    if not os.path.isfile(filename):
        raise Exception("Not exists file %s" %filename)
    url_list = ReadFile(filename)

    #每次POST提交url的條數
    MAX_REQUEST_SIZE = 800
    while url_list:
        batch = []
        batch_size = 0

        while url_list and batch_size < MAX_REQUEST_SIZE:
            next_url = url_list.pop()
            batch.append(next_url)
            batch_size += 1
        if batch:
            response = api.postPurgeRequest(batch)
            if response["httpStatus"] != 201:
                # 將本次請求url返回到總url列表中，以便稍后在試
                url_list.extend(batch)
                #速率限制
                if response["httpStatus"] == 507:
                    details = response.json().get('detail', '<response did not contain "detail">')
                    print('Will retry request in 1s seconds due to API rate-limit: %s,Try again now.'%details)
                    logger.info('Will retry request in 1s seconds due to API rate-limit: %s,Try again now.'%details)
                    time.sleep(1)
                # 針對速率限制以外的錯誤  拋出
                if response["httpStatus"] != 507:
                    details = response.json().get('detail', '<response did not contain "detail">')
                    print("{status:Failed,detail:%s}"%details)
                    logger.info("{status:Failed,detail:%s}"%details)
                    response.raise_for_status()
            else:
                logger.info("{status:Success,supportId:%s,purgeId:%s,queue:%s}"%(response["supportId"],response["purgeId"],json.dumps(batch)))


# 依賴包：

[root@jenkins:/usr/local/worksh/jeninks_task/akamai_api]# cat requirement.txt 
asn1crypto==0.24.0
certifi==2019.6.16
cffi==1.12.3
chardet==3.0.4
configparser==3.7.4
cryptography==2.7
edgegrid-python==1.1.1
idna==2.8
ndg-httpsclient==0.5.1
pyasn1==0.4.6
pycparser==2.19
pyOpenSSL==19.0.0
requests==2.22.0
six==1.12.0
urllib3==1.25.3


[root@jenkins:/usr/local/worksh/jeninks_task/akamai_api]# cat lib/http_calls.py
#!/usr/bin/env python


# Python edgegrid module
""" Copyright 2015 Akamai Technologies, Inc. All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

 You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""
import sys

if sys.version_info[0] >= 3:
    # python3
    from urllib import parse
else:
    # python2.7
    import urlparse as parse

import logging, json

logger = logging.getLogger(__name__)


class EdgeGridHttpCaller():
    def __init__(self, session, debug, verbose, baseurl):
        self.debug = debug
        self.verbose = verbose
        self.session = session
        self.baseurl = baseurl
        return None

    def urlJoin(self, url, path):
        return parse.urljoin(url, path)

    def getResult(self, endpoint, parameters=None):
        path = endpoint
        endpoint_result = self.session.get(parse.urljoin(self.baseurl, path), params=parameters)
        if self.verbose: print(">>>\n" + json.dumps(endpoint_result.json(), indent=2) + "\n<<<\n")
        status = endpoint_result.status_code
        if self.verbose: print("LOG: GET %s %s %s" % (endpoint, status, endpoint_result.headers["content-type"]))
        self.httpErrors(endpoint_result.status_code, path, endpoint_result.json())
        return endpoint_result.json()

    def httpErrors(self, status_code, endpoint, result):
        if not isinstance(result, list):
            details = result.get('detail') or result.get('details') or ""
        else:
            details = ""
        if status_code == 403:
            error_msg = "ERROR: Call to %s failed with a 403 result\n" % endpoint
            error_msg += "ERROR: This indicates a problem with authorization.\n"
            error_msg += "ERROR: Please ensure that the credentials you created for this script\n"
            error_msg += "ERROR: have the necessary permissions in the Luna portal.\n"
            error_msg += "ERROR: Problem details: %s\n" % details
            exit(error_msg)

        if status_code in [400, 401]:
            error_msg = "ERROR: Call to %s failed with a %s result\n" % (endpoint, status_code)
            error_msg += "ERROR: This indicates a problem with authentication or headers.\n"
            error_msg += "ERROR: Please ensure that the .edgerc file is formatted correctly.\n"
            error_msg += "ERROR: If you still have issues, please use gen_edgerc.py to generate the credentials\n"
            error_msg += "ERROR: Problem details: %s\n" % result
            exit(error_msg)

        if status_code in [404]:
            error_msg = "ERROR: Call to %s failed with a %s result\n" % (endpoint, status_code)
            error_msg += "ERROR: This means that the page does not exist as requested.\n"
            error_msg += "ERROR: Please ensure that the URL you're calling is correctly formatted\n"
            error_msg += "ERROR: or look at other examples to make sure yours matches.\n"
            error_msg += "ERROR: Problem details: %s\n" % details
            exit(error_msg)

        error_string = None
        if "errorString" in result:
            if result["errorString"]:
                error_string = result["errorString"]
        else:
            for key in result:
                if type(key) is not str or isinstance(result, dict) or not isinstance(result[key], dict):
                    continue
                if "errorString" in result[key] and type(result[key]["errorString"]) is str:
                    error_string = result[key]["errorString"]
        if error_string:
            error_msg = "ERROR: Call caused a server fault.\n"
            error_msg += "ERROR: Please check the problem details for more information:\n"
            error_msg += "ERROR: Problem details: %s\n" % error_string
            exit(error_msg)

    def postResult(self, endpoint, body, parameters=None):
        headers = {'content-type': 'application/json'}
        path = endpoint
        endpoint_result = self.session.post(parse.urljoin(self.baseurl, path), data=body, headers=headers,
                                            params=parameters)
        status = endpoint_result.status_code
        if self.verbose: print("LOG: POST %s %s %s" % (path, status, endpoint_result.headers["content-type"]))
        if status == 204:
            return {}
        self.httpErrors(endpoint_result.status_code, path, endpoint_result.json())

        if self.verbose: print(">>>\n" + json.dumps(endpoint_result.json(), indent=2) + "\n<<<\n")
        return endpoint_result.json()

    def postFiles(self, endpoint, file):
        path = endpoint
        endpoint_result = self.session.post(parse.urljoin(self.baseurl, path), files=file)
        status = endpoint_result.status_code
        if self.verbose: print("LOG: POST FILES %s %s %s" % (path, status, endpoint_result.headers["content-type"]))
        if status == 204:
            return {}
        self.httpErrors(endpoint_result.status_code, path, endpoint_result.json())

        if self.verbose: print(">>>\n" + json.dumps(endpoint_result.json(), indent=2) + "\n<<<\n")
        return endpoint_result.json()

    def putResult(self, endpoint, body, parameters=None):
        headers = {'content-type': 'application/json'}
        path = endpoint

        endpoint_result = self.session.put(parse.urljoin(self.baseurl, path), data=body, headers=headers,
                                           params=parameters)
        status = endpoint_result.status_code
        if self.verbose: print("LOG: PUT %s %s %s" % (endpoint, status, endpoint_result.headers["content-type"]))
        if status == 204:
            return {}
        if self.verbose: print(">>>\n" + json.dumps(endpoint_result.json(), indent=2) + "\n<<<\n")
        return endpoint_result.json()

    def deleteResult(self, endpoint):
        endpoint_result = self.session.delete(parse.urljoin(self.baseurl, endpoint))
        status = endpoint_result.status_code
        if self.verbose: print("LOG: DELETE %s %s %s" % (endpoint, status, endpoint_result.headers["content-type"]))
        if status == 204:
            return {}
        if self.verbose: print(">>>\n" + json.dumps(endpoint_result.json(), indent=2) + "\n<<<\n")
        return endpoint_result.json()

 

# 報錯

warning: inexact rename detection was skipped due to too many files.
warning: you may want to set your diff.renameLimit variable to at least 3074 and retry the command.
Traceback (most recent call last):
  File "/usr/local/worksh/jeninks_task/akamai_api/akamai_api.py", line 109, in <module>
    response = api.postPurgeRequest(batch)
  File "/usr/local/worksh/jeninks_task/akamai_api/akamai_api.py", line 48, in postPurgeRequest
    purge_post_result = httpCaller.postResult('/ccu/v3/%s/url/%s' % (self.action, self.network), json.dumps(purge_obj))
  File "/usr/local/worksh/jeninks_task/akamai_api/lib/http_calls.py", line 112, in postResult
    self.httpErrors(endpoint_result.status_code, path, endpoint_result.json())
  File "/usr/local/worksh/jeninks_task/akamai_api_venv/lib/python3.6/site-packages/requests/models.py", line 897, in json
    return complexjson.loads(self.text, **kwargs)
  File "/usr/local/python3/lib/python3.6/json/__init__.py", line 354, in loads
    return _default_decoder.decode(s)
  File "/usr/local/python3/lib/python3.6/json/decoder.py", line 339, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/local/python3/lib/python3.6/json/decoder.py", line 357, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
刷新CDN失敗

 

修改 MAX_REQUEST_SIZE = 200，每次批量提交800條，被cdn拒絕，修改為200后問題解決