Python 自动获取License文件

需求：

公司做了几个nodejs项目，牵扯到License问题。希望做 一个  把这个项目中引用到的dependency中的license 的文件内容 整合到一个TXT文档里。

for example:

根目录下的 package.json 中dependencies{ "protobufjs": "6.10.1"}

则需要查找protobufjs 中的package.json 中的 

"dependencies": {
"@protobufjs/aspromise": "^1.1.2",
"@protobufjs/base64": "^1.1.2",
"@protobufjs/codegen": "^2.0.4",
"@protobufjs/eventemitter": "^1.1.0",
"@protobufjs/fetch": "^1.1.0",
"@protobufjs/float": "^1.0.2",
"@protobufjs/inquire": "^1.1.0",
"@protobufjs/path": "^1.1.2",
"@protobufjs/pool": "^1.1.0",
"@protobufjs/utf8": "^1.1.0",
"@types/long": "^4.0.1",
"@types/node": "^13.7.0",
"long": "^4.0.0"
}

在去查找@protobufjs/aspromise下的package.json ，直到找不到dependencies为止，

然后把这些dependencies对应的license内容整理到TXT文件中去。

** 不可重复

 

解题思路：

我比较愚钝，这个功能做了好几天，问题的难点在于如何查找所有引用到的依赖库。

思考一：递归思想

1. 先解析根目录下的package.json 文件，查询到用的dependencies

2.用os.walk(path) 循环的方式获取到遍历根目录下方与查询到的dependencies一致的文件夹A

3.在文件夹A的路径下，判断license文件是否存在，若存在则读取license文件到TXT中，如果不存在，在TXT中写入dependency的名字，注明license 未查询到。

4.在文件夹A的路径下，读取package.json 文件，读取解析dependencies（循环过程1，2，3）

在文件夹少的情况下，可行，过大的时候，× 不可行。

 

思考二：

1. 遍历文件夹下的所有package.json 文件，获取所有package.json 文件的路径，作为一个list存储下来 pkgDirList（如果出现重复的需要对其进行解析，争取排除重复项）

2. 以pkgDirList为输入，计算每一个dependency出现的次数，和dependency对应的路径，比如dictCountDir["axios"]=1，dictNameDir["axios"] ="C:xxx\\mxxx\node_modules\axios"

2. 以根目录中的pakage.json 文件中的dependencies为入口，然后循环遍历pkgDirList，在pkgDirList中找到对应的文件路径，

3. 读取package.json 文件，解析后在去pkgDirList 匹配查找对应路径，以此下去, 直到查完所有的package.json 文件中的dependency，返回一个包含所有dependencies的list, depList

4. 将获取到的所有dependencies的depList 中的元素作为 dictNameDir的key查找对应的license 路径，

4.1 如果license 文件存在，根据license路径读取文件。写入TXT文档。

4.2 如果license文件不存在，再去判断license内容是否存在于readme.md 文件中，

　　　　4.2.1如果存在，加入提醒信息，请手动copy，

　　　　4.2.2如果不存在，判断是不是MIT license，如果是MIT license 导入MIT license 模板。

　　　　4.2.3 如果都不是，备注找不到。

 

源码：

 

import os
import sys
import json
import time


#return dependencies string
#for example:
# '"axios": "^0.19.2","myjs-common": "^1.0.6","oneport": "^1.0.2" '
def readPackageJson(filename):
    dependenceStr = ""
    try:
        if os.path.exists(filename):
             with open(filename,"r",encoding='utf8') as dependencies:
                packageJson = json.load(dependencies)
                if "dependencies" in packageJson and packageJson["dependencies"] != {}:
                        dependenceStr = str(packageJson["dependencies"]).replace("{","").replace("}","")
        return dependenceStr
    except Exception as e:
        print(e)

 
#for example:
# '"license": "MIT"' in package.json file 
#  if keyName == license, valueStr==MIT;
#  if keyName = homepage, valueStr=https://github.com/indutny/node-spdy
def GetPackageJsonInfo(filePath,keyName):
    valueStr = ""
    try:
        filePath = os.path.join("%s%s" % (filePath,"\\package.json"))
        if os.path.exists(filePath):
             with open(filePath,"r",encoding='utf8') as pkgJson:
                packageJson = json.load(pkgJson)
                if keyName in packageJson:
                    valueStr = str(packageJson[keyName])
        return valueStr
    except Exception as e:
        print(e)

# Whether the readme.md file contains MIT license content, 
# it returns True and file dir, but does not return False.
def readReadmefile(filePath):
    fileNames = ["\\readme.md","\\README.md","\\Readme.md","\\readme.txt"]
    for fileName in fileNames:
        filePath = os.path.join("%s%s"% (filePath,fileName))
        if os.path.exists(filePath):
            with open(filePath,"r",encoding="utf8") as readMe:
                if "copies or substantial portions of the Software." in readMe.read():
                    return True
                else:
                    return False



#depStr is dependencies from package.json file,like '"axios": "^0.19.2","myjs-common": "^1.0.6","oneport": "^1.0.2" '
#return a dependencies list, for example:["axios","myjs-common","oneport"]
def ParsingPackageJson(depStr):
        depList = []
        for dep in depStr.split(","):
            if len(dep.split(":")) == 2:
                depList.append(dep.split(":")[0].strip().replace("'",""))
        return sorted(list(set(depList)))

def getLicenseFilePath(filepath):
        licensefilename = ["LICENSE","LICENCE","LICENSE-MIT","LICENCE-MIT"]
        licensepath = None
        stopCircle = False
        count = 0
        for dirpath,dirnames,filenames in os.walk(filepath): 
            for filename in filenames:
                count = count + 1
                fileNameTemp = filename.upper().split(".")
                if (len(fileNameTemp) >= 2 and (fileNameTemp[-1] != "JS" or fileNameTemp[-1] != "HTML")) or len(fileNameTemp) == 1:
                    if fileNameTemp[0] in licensefilename :
                        licensepath = os.path.join('%s%s' % (filepath, "\\" + filename))
                        stopCircle = True
                        break
                        #print(filename)
            if stopCircle or count == len(filenames):
                break
        return licensepath

#Get the dependencies in the package.json file in the root directory
# return dependencies info str
def getRootPackageJson(rootPath):
    depStr = ""
    findPackageJson = False
    for fileNames in os.walk(rootPath):
        if "node_modules" in fileNames[1]:
            global nodeModulesDir
            nodeModulesDir =  os.path.join('%s%s' % (fileNames[0], "\\node_modules"))
        for pfile in fileNames[-1]:
            if pfile == "package.json":
                packageJsonPath =  os.path.join('%s%s' % (fileNames[0], "\\package.json"))
                depStr = readPackageJson(packageJsonPath)
                findPackageJson = True
                break
        break
        if findPackageJson:
            print("get pacakageJson")
        else:
            print("No pacakageJson")
    return depStr

# DepDir,pkg file dir
def getDependencyName(DepDir):
    if "@" in DepDir:
        dependenceName ="@" + (DepDir.split("@")[-1]).replace("\\","/")
    else:
        dependenceName = DepDir.split("\\")[-1]
    return dependenceName

# Find the dir where all package.json files are located,
# Return a list containing all package.json dir  
def getAllPackageJsonDir(Dir):
    packageJsonDir = [] 
    depName = []   
    for dirs,fileNames,files in os.walk(Dir):
        for file in files:
            if file=="package.json":
                dependenceName = getDependencyName(dirs)
                if dependenceName not in depName:
                    depName.append(dependenceName)
                    packageJsonDir.append(dirs)
                else:
                    # The dependencies in package.json in the dependency folders of the same name under different folders may be different
                    # for example:
                    # G:...\..\node_modules\body-parser 
                    # G:...\..\node_modules\@types\body-parser 
                    for pjsonDir in packageJsonDir:
                        if pjsonDir.split("\\")[-1]==dirs.split("\\")[-1]:
                            pjsonPath = os.path.join("%s%s" % (pjsonDir,"\\package.json")) 
                            pjsonPathNew = os.path.join("%s%s" % (dirs,"\\package.json"))
                            pjsonStr = readPackageJson(pjsonPath) 
                            pjsonStrNew = readPackageJson(pjsonPathNew) 
                            pName = ParsingPackageJson(pjsonStr)
                            pNameNew = ParsingPackageJson(pjsonStrNew)
                            if pName!=pNameNew:
                                packageJsonDir.append(dirs)
                                #saveInfoToTxt(ResultInfoPath,dirs)
    print("all package.json file dir count:",len(packageJsonDir))
    return packageJsonDir

# for example: dictCountDir["axios"]=1，dictNameDir["axios"] =r"C:...\meetingServer_1.0.8\node_modules\axios"
# return dictCountDir,dictNameDir
def getPkgDirDict(pacakageDir,rootDir):
    #dictCountDir  key: dependence name, value: dir count of same dependencies
    dictCountDir={}
    #dictNameDir  key: dependence name，value: dependence dir
    dictNameDir={}
    for pkgDir in pacakageDir:
        if pkgDir == rootDir:
            continue
        pkgDirName = getDependencyName(pkgDir)
        countDir=0
        if  pkgDirName not in dictNameDir.keys():
            dictNameDir[pkgDirName] = pkgDir
            dictCountDir[pkgDirName] = countDir+1
        else:
            dictCountDir[pkgDirName] = dictCountDir[pkgDirName] +1
    return dictNameDir,dictCountDir

# Read the package.json file under pkgPath, if dependencies is not empty,
# analyze whether the dependence is in the deplist, if not, add it to the deplist, and return to the deplist.
# pkgName:dependence name, depList:dependence list, pkgPath: package.json file path
# return dependency list
def getSubDepList(pkgName,deplist,pkgPath):
    pkgJson = readPackageJson(pkgPath)
    if pkgJson:
         pkgDep = ParsingPackageJson(pkgJson)
         for subDep in pkgDep:
            if subDep not in deplist:
                deplist.append(subDep)
         info = "pkgName:"+ pkgName + "  pkgPath:"+ pkgPath +"\r pkgJson:"+ str(pkgJson)+"\r\n"
         saveInfoToTxt(ResultInfoPath,info)
    return deplist

#Query all dependencies according to the dependencies value in package.json 
#Return a list of all dependencies
def getDepList(rootDepStr,pacakageDir,dirDict):
    deplist=[]
    dictNameDirs=dirDict[0]
    dictCountDir = dirDict[1]
    if rootDepStr:
        # 根据packageDir 分析依赖库
        deplist = ParsingPackageJson(rootDepStr)
        for dep in deplist:
            for pkgDir in pacakageDir:
                pkgDirName = getDependencyName(pkgDir)
                if dep==pkgDirName:
                    pkgPath =  os.path.join('%s%s' % (pkgDir, "\\package.json")) 
                    getSubDepList(dep,deplist,pkgPath)
                    if dictCountDir[pkgDirName] ==1:
                        break
        saveInfoToTxt(ResultInfoPath,"deplist count:"+ str(len(deplist))+ "\r" +str(deplist))
        dcount = len(deplist)
        print("dependence count:",str(dcount))

    return deplist

# test function, find all depencies
def getAllDepList(pacakageDir):
    depStr = ""
    for dir in pacakageDir:
        pkgPath =  os.path.join('%s%s' % (dir, "\\package.json")) 
        pJson = readPackageJson(pkgPath)
        if pJson:
            depStr =  depStr +  pJson 
        depStr = depStr +","
    allDep = ParsingPackageJson(depStr)
    return allDep

def getLicenses(rootdir):

    pkgDirList = getAllPackageJsonDir(rootdir)
    dirDict = getPkgDirDict(pkgDirList,rootdir)
    rootDepStr = getRootPackageJson(rootdir)
    deplist = getDepList(rootDepStr,pkgDirList,dirDict)

    licenseTypeDict= {"FindInReadme":0, "AddMITLicenseTemp":0,"NotMITLicense":0,"Others":0}
    # According to find all deplist, find the license file under the corresponding file.
    licenseNo = 0 # license number in AllLicenses.txt file
    lNoFindCount = 0  # Total number of license files not found
    LicenseFileNotFind=[] #
    depFileNotFind=[] # The file directory where the dependency file is not found
    for depName in deplist:
        licenseNo = licenseNo + 1
        dictNameDir = dirDict[0]
        if depName in dictNameDir.keys():
            licensepath = getLicenseFilePath(dictNameDir[depName])
            if licensepath is None:
                licensepath = None
                LicenseFileNotFind.append(depName)
                lNoFindCount = lNoFindCount+1
                res = AddLicenseTemp(licenseNo,dictNameDir[depName],depName)
                licenseTypeDict[res] = licenseTypeDict[res]+1
            else:
                readLicense(licenseNo,licensepath,depName) 
        else:
            lNoFindCount = lNoFindCount+1
            depFileNotFind.append(depName)
            res = AddLicenseTemp(licenseNo,None,depName)
            licenseTypeDict[res] = licenseTypeDict[res]+1

    notFindDepCount = str(len(depFileNotFind))
    notFindLinceseCount = str(len(LicenseFileNotFind))

    print("Not find Dependence file :",notFindDepCount)
    print("Not find license file:", notFindLinceseCount)
    print("License file Not find count:",str(lNoFindCount))
    for key in licenseTypeDict:
        print(key,licenseTypeDict[key])
    saveInfoToTxt(ResultInfoPath,"\r\n Info of license file is found"+ str(licenseTypeDict))
    saveInfoToTxt(ResultInfoPath,"\r\n Not find Dependence dir count:"+ notFindDepCount +" \r"+ str(depFileNotFind))
    saveInfoToTxt(ResultInfoPath,"\r\n Not find license file count: "+ notFindLinceseCount + "\r"+ str(LicenseFileNotFind))
    saveInfoToTxt(ResultInfoPath,"\r\n total license file Not find count:"+str(lNoFindCount))

def saveInfoToTxt(savePath,line):
     with open(savePath,"a+") as f:
        f.write(str(line)) 
        f.write('\r')

# no:license number
# filepath: license file path
# depName: dependence name
# There is a license file under dir, read the license content
def readLicense(no,filepath,depName):
    with open(ResultLincensePath,"a+",encoding="utf8") as f:
        f.write("\r==================================\r")
        f.write(str(no) + ") " + depName)
        f.writelines("\r==================================\r")
        with open(filepath,"r", encoding="utf8") as licensefile:     
            f.write(licensefile.read()) 
            f.write('\r')

# There is no license file under dir, add license template
# 
def AddLicenseTemp(no,filepath,depName):
    if filepath != None:
        isMITlicense = readReadmefile(filepath)
        with open(ResultLincensePath,"a+",encoding="utf8") as f:
            f.write("\r==================================\r")
            f.write(str(no) + ") " + depName)
            f.writelines("\r==================================\r")
        if isMITlicense:
            saveInfoToTxt(ResultInfoPath,"** %s MIT License in readme.md file,Please paste manually. %s \r" % (depName,filepath))
            with open(ResultLincensePath,"a+",encoding="utf8") as f:
                f.write("MIT License in readme.md file,Please paste manually! " )
                f.write('\r')
            return "FindInReadme"
        else:
            license = GetPackageJsonInfo(filepath,"license")
            homepage = GetPackageJsonInfo(filepath,"homepage")
            if homepage:
                with open(ResultLincensePath,"a+",encoding="utf8") as f:
                    f.write("Homepage: %s%s" % (homepage,"\r"))
            if license=="MIT":
                with open(ResultLincensePath,"a+",encoding="utf8") as f:
                    with open(lincenseTempPath,"r", encoding="utf8") as licenseTemp:
                        f.write(licenseTemp.read()) 
                        f.write('\r')
                saveInfoToTxt(ResultInfoPath,">> %s Add MIT License Template. " % depName)
                return "AddMITLicenseTemp"
            else:
                #with open(ResultLincensePath,"a+",encoding="utf8") as f:
                #    f.write("License:%s" % license )
                saveInfoToTxt(ResultInfoPath, "@@ %s is not an MIT License, its license is: %s" % (depName,license))
                return "NotMITLicense"
    else:
         with open(ResultLincensePath,"a+",encoding="utf8") as f:
            f.write("\r==================================\r")
            f.write(str(no) + ") " + depName)
            f.writelines("\r==================================\r")
            f.write("Nothing find !" )
            saveInfoToTxt(ResultInfoPath,"/(ㄒoㄒ)/~~  Nothing find ! %s" % depName)
            return "Others"

# Initialization Result path
def inite():
    global ResultDefaultPath,ResultLincensePath,ResultInfoPath,lincenseTempPath
    ResultDefaultPath = os.path.join(os.getcwd(),"Results")
    if not os.path.exists(ResultDefaultPath):
        os.mkdir(ResultDefaultPath)
    ResultLincensePath = os.path.join('%s%s' % (ResultDefaultPath, "\\AllLicenses.txt"))
    ResultInfoPath = os.path.join('%s%s' % (ResultDefaultPath, "\\AllInfo.txt"))
    lincenseTempPath = os.path.join('%s%s' % (ResultDefaultPath, "\\MITLicenseTemp.txt"))
    if not os.path.exists(lincenseTempPath):
        print("license template file in Result dir is not exists.")
        time.sleep(20)
        exit(0)
    if os.path.exists(ResultLincensePath):
        os.remove(ResultLincensePath)
    if os.path.exists(ResultInfoPath):
        os.remove(ResultInfoPath)

if __name__ == '__main__':

    inite()
    rootDir = os.getcwd()
    #os.getcwd()
    print("***Root File Path：",rootDir)
    print("Begin Time:",time.strftime('%Y/%m/%d %H:%M:%S'))
    saveInfoToTxt(ResultInfoPath,"Begin Time:"+time.strftime('%Y/%m/%d %H:%M:%S'))
    getLicenses(rootDir)
    print("All finished:",time.strftime('%Y/%m/%d %H:%M:%S'))
    saveInfoToTxt(ResultInfoPath,"All finished:"+ time.strftime('%Y/%m/%d %H:%M:%S'))
    print("\r\n ***Complete! Please refer Results folder.***")
    input("Press any key to close.")