SQLmap源碼分析之框架初始化(一)

SQLmap是現在搞web人手一個的注入神器，不僅包含了主流數據庫的SQL注入檢測，而且包含提權以及后滲透模塊。基於python2.x開發而成，使用方便。所以研究web安全少不了分析源碼，學習代碼的同時，也可以學習先進的漏洞檢測技術。多的不多說，咱們來分析一下源碼。

 

使用的工具如下：

IDE： sublime text
SQLmap源碼：https://github.com/sqlmapproject/sqlmap
當前分析版本： 1.1.2.5

 

0x00 從入口文件開始

我們在拿到源代碼以后，先拉進sublime text中然后開始從sqlmap.py的入口文件開始分析。在入口函數調用之前，首先從lib目錄下引入了很多的文件用作系統的初始化操作。然后我們來看入口文件main()函數中都做了哪些事情？

def main():
    """
    Main function of sqlmap when running from command line.
    """

    try:
        checkEnvironment()

        setPaths(modulePath())

        banner()

        # Store original command line options for possible later restoration
        cmdLineOptions.update(cmdLineParser().__dict__)

        initOptions(cmdLineOptions)

        if hasattr(conf, "api"):
            # heavy imports
            from lib.utils.api import StdDbOut
            from lib.utils.api import setRestAPILog

            # Overwrite system standard output and standard error to write
            # to an IPC database
            sys.stdout = StdDbOut(conf.taskid, messagetype="stdout")
            sys.stderr = StdDbOut(conf.taskid, messagetype="stderr")
            setRestAPILog()

        conf.showTime = True
        dataToStdout("[!] legal disclaimer: %s\n\n" % LEGAL_DISCLAIMER, forceOutput=True)
        dataToStdout("[*] starting at %s\n\n" % time.strftime("%X"), forceOutput=True)

        init()

        if conf.profile:
            profile()
        elif conf.smokeTest:
            smokeTest()
        elif conf.liveTest:
            liveTest()
        else:
            try:
                start()
            except thread.error as ex:
                if "can't start new thread" in getSafeExString(ex):
                    errMsg = "unable to start new threads. Please check OS (u)limits"
                    logger.critical(errMsg)
                    raise SystemExit
                else:
                    raise

    except SqlmapUserQuitException:
        errMsg = "user quit"
        try:
            logger.error(errMsg)
        except KeyboardInterrupt:
            pass

    except (SqlmapSilentQuitException, bdb.BdbQuit):
        pass

    except SqlmapShellQuitException:
        cmdLineOptions.sqlmapShell = False

    except SqlmapBaseException as ex:
        errMsg = getSafeExString(ex)
        try:
            logger.critical(errMsg)
        except KeyboardInterrupt:
            pass
        raise SystemExit

    except KeyboardInterrupt:
        print

        errMsg = "user aborted"
        try:
            logger.error(errMsg)
        except KeyboardInterrupt:
            pass

    except EOFError:
        print
        errMsg = "exit"

        try:
            logger.error(errMsg)
        except KeyboardInterrupt:
            pass

    except SystemExit:
        pass

    except:
        print
        errMsg = unhandledExceptionMessage()
        excMsg = traceback.format_exc()

        try:
            if not checkIntegrity():
                errMsg = "code integrity check failed (turning off automatic issue creation). "
                errMsg += "You should retrieve the latest development version from official GitHub "
                errMsg += "repository at '%s'" % GIT_PAGE
                logger.critical(errMsg)
                print
                dataToStdout(excMsg)
                raise SystemExit

            elif "tamper/" in excMsg:
                logger.critical(errMsg)
                print
                dataToStdout(excMsg)
                raise SystemExit

            elif "MemoryError" in excMsg:
                errMsg = "memory exhaustion detected"
                logger.error(errMsg)
                raise SystemExit

            elif any(_ in excMsg for _ in ("No space left", "Disk quota exceeded")):
                errMsg = "no space left on output device"
                logger.error(errMsg)
                raise SystemExit

            elif all(_ in excMsg for _ in ("No such file", "_'", "self.get_prog_name()")):
                errMsg = "corrupted installation detected ('%s'). " % excMsg.strip().split('\n')[-1]
                errMsg += "You should retrieve the latest development version from official GitHub "
                errMsg += "repository at '%s'" % GIT_PAGE
                logger.error(errMsg)
                raise SystemExit

            elif "Read-only file system" in excMsg:
                errMsg = "output device is mounted as read-only"
                logger.error(errMsg)
                raise SystemExit

            elif "OperationalError: disk I/O error" in excMsg:
                errMsg = "I/O error on output device"
                logger.error(errMsg)
                raise SystemExit

            elif "_mkstemp_inner" in excMsg:
                errMsg = "there has been a problem while accessing temporary files"
                logger.error(errMsg)
                raise SystemExit

            elif "can't start new thread" in excMsg:
                errMsg = "there has been a problem while creating new thread instance. "
                errMsg += "Please make sure that you are not running too many processes"
                if not IS_WIN:
                    errMsg += " (or increase the 'ulimit -u' value)"
                logger.error(errMsg)
                raise SystemExit

            elif all(_ in excMsg for _ in ("pymysql", "configparser")):
                errMsg = "wrong initialization of pymsql detected (using Python3 dependencies)"
                logger.error(errMsg)
                raise SystemExit

            elif "bad marshal data (unknown type code)" in excMsg:
                match = re.search(r"\s*(.+)\s+ValueError", excMsg)
                errMsg = "one of your .pyc files are corrupted%s" % (" ('%s')" % match.group(1) if match else "")
                errMsg += ". Please delete .pyc files on your system to fix the problem"
                logger.error(errMsg)
                raise SystemExit

            elif "valueStack.pop" in excMsg and kb.get("dumpKeyboardInterrupt"):
                raise SystemExit

            for match in re.finditer(r'File "(.+?)", line', excMsg):
                file_ = match.group(1)
                file_ = os.path.relpath(file_, os.path.dirname(__file__))
                file_ = file_.replace("\\", '/')
                file_ = re.sub(r"\.\./", '/', file_).lstrip('/')
                excMsg = excMsg.replace(match.group(1), file_)

            errMsg = maskSensitiveData(errMsg)
            excMsg = maskSensitiveData(excMsg)

            if hasattr(conf, "api"):
                logger.critical("%s\n%s" % (errMsg, excMsg))
            else:
                logger.critical(errMsg)
                kb.stickyLevel = logging.CRITICAL
                dataToStdout(excMsg)
                createGithubIssue(errMsg, excMsg)

        except KeyboardInterrupt:
            pass

    finally:
        kb.threadContinue = False

        if conf.get("showTime"):
            dataToStdout("\n[*] shutting down at %s\n\n" % time.strftime("%X"), forceOutput=True)

        kb.threadException = True

        if kb.get("tempDir"):
            for prefix in (MKSTEMP_PREFIX.IPC, MKSTEMP_PREFIX.TESTING, MKSTEMP_PREFIX.COOKIE_JAR, MKSTEMP_PREFIX.BIG_ARRAY):
                for filepath in glob.glob(os.path.join(kb.tempDir, "%s*" % prefix)):
                    try:
                        os.remove(filepath)
                    except OSError:
                        pass
            if not filter(None, (filepath for filepath in glob.glob(os.path.join(kb.tempDir, '*')) if not any(filepath.endswith(_) for _ in ('.lock', '.exe', '_')))):
                shutil.rmtree(kb.tempDir, ignore_errors=True)

        if conf.get("hashDB"):
            try:
                conf.hashDB.flush(True)
            except KeyboardInterrupt:
                pass

        if cmdLineOptions.get("sqlmapShell"):
            cmdLineOptions.clear()
            conf.clear()
            kb.clear()
            main()

        if hasattr(conf, "api"):
            try:
                conf.databaseCursor.disconnect()
            except KeyboardInterrupt:
                pass

        if conf.get("dumper"):
            conf.dumper.flush()

        # short delay for thread finalization
        try:
            _ = time.time()
            while threading.activeCount() > 1 and (time.time() - _) > THREAD_FINALIZATION_TIMEOUT:
                time.sleep(0.01)
        except KeyboardInterrupt:
            pass
        finally:
            # Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program
            if threading.activeCount() > 1:
                os._exit(0)

 

我們可以看到這里，首先調用了checkEnvironment()函數，根據名字我們知道這個函數的作用是檢測環境。我們跟進來看這個函數：

def checkEnvironment():
    try:
        os.path.isdir(modulePath())
    except UnicodeEncodeError:
        errMsg = "your system does not properly handle non-ASCII paths. "
        errMsg += "Please move the sqlmap's directory to the other location"
        logger.critical(errMsg)
        raise SystemExit

    if distutils.version.LooseVersion(VERSION) < distutils.version.LooseVersion("1.0"):
        errMsg = "your runtime environment (e.g. PYTHONPATH) is "
        errMsg += "broken. Please make sure that you are not running "
        errMsg += "newer versions of sqlmap with runtime scripts for older "
        errMsg += "versions"
        logger.critical(errMsg)
        raise SystemExit

    # Patch for pip (import) environment
    if "sqlmap.sqlmap" in sys.modules:
        for _ in ("cmdLineOptions", "conf", "kb"):
            globals()[_] = getattr(sys.modules["lib.core.data"], _)

        for _ in ("SqlmapBaseException", "SqlmapShellQuitException", "SqlmapSilentQuitException", "SqlmapUserQuitException"):
            globals()[_] = getattr(sys.modules["lib.core.exception"], _)

調用了module()函數並且判斷是否是一個正確的路徑，如果不是的話，那么將會打印錯誤信息並且拋出一個異常終止程序繼續運行。

我們繼續來看module()函數中做了一些什么：

def modulePath():
    """
    This will get us the program's directory, even if we are frozen
    using py2exe
    """

    try:
        _ = sys.executable if weAreFrozen() else __file__  #如果用py2exe封裝，那么_為python的絕對路徑否則就是當前文件名也就是sqlmap.py
    except NameError:
        _ = inspect.getsourcefile(modulePath)

    return getUnicode(os.path.dirname(os.path.realpath(_)), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)

我們在注釋中可以看到是為了獲取程序所在的目錄。為了防止亂碼，返回unicode編碼的路徑。

getUnicode()函數在這里：sqlmap\lib\core\common.py。這里就不貼代碼了。

然后checkEnvironment()判斷版本。接着判斷sqlmap.sqlmap是否已經加載，如果加載，那么就獲取到cmdLineOptions, conf, kb幾個屬性並且把它們作為全局變量。

接下來，setPaths(modulePath())設置了一下系統各個部分的絕對路徑，並且判斷.txt, .xml, .zip為擴展名的文件是否存在並且是否可讀。

這里我們來思考一個問題，為什么全局要用絕對路徑呢？做過開發的同學就知道了，用絕對路徑可以避免很多不必要的麻煩，比如說包含文件時候，用相對路徑，互相包含，最后越搞越亂，一旦換了一個目錄，就會出問題。也不方便日后的維護。用絕對路徑，所有的調用全部放在主入口文件，這樣單一入口的原則使得系統不僅調用方便，而且看起來還緊湊有序。

然后就是打印banner的信息。這里還有一個值得注意的點就是AttribDict這個數據類型。是這樣定義的：

 

class AttribDict(dict):
    """
    This class defines the sqlmap object, inheriting from Python data
    type dictionary.

    >>> foo = AttribDict()
    >>> foo.bar = 1
    >>> foo.bar
    1
    """

    def __init__(self, indict=None, attribute=None):
        if indict is None:
            indict = {}

        # Set any attributes here - before initialisation
        # these remain as normal attributes
        self.attribute = attribute
        dict.__init__(self, indict)
        self.__initialised = True

        # After initialisation, setting attributes
        # is the same as setting an item

    def __getattr__(self, item):
        """
        Maps values to attributes
        Only called if there *is NOT* an attribute with this name
        """

        try:
            return self.__getitem__(item)
        except KeyError:
            raise AttributeError("unable to access item '%s'" % item)

    def __setattr__(self, item, value):
        """
        Maps attributes to values
        Only if we are initialised
        """

        # This test allows attributes to be set in the __init__ method
        if "_AttribDict__initialised" not in self.__dict__:
            return dict.__setattr__(self, item, value)

        # Any normal attributes are handled normally
        elif item in self.__dict__:
            dict.__setattr__(self, item, value)

        else:
            self.__setitem__(item, value)

    def __getstate__(self):
        return self.__dict__

    def __setstate__(self, dict):
        self.__dict__ = dict

    def __deepcopy__(self, memo):
        retVal = self.__class__()
        memo[id(self)] = retVal

        for attr in dir(self):
            if not attr.startswith('_'):
                value = getattr(self, attr)
                if not isinstance(value, (types.BuiltinFunctionType, types.FunctionType, types.MethodType)):
                    setattr(retVal, attr, copy.deepcopy(value, memo))

        for key, value in self.items():
            retVal.__setitem__(key, copy.deepcopy(value, memo))

        return retVal

 

繼承了內置的dict，並且重寫了一些方法。然后就可以這樣去訪問鍵值對：var.key。感慨 一下，好牛！！！

 

0x01  獲取命令行參數選項

這里主要使用了optparse這個函數庫。python中十分好用的一個命令行工具。具體可以參考這里：https://docs.python.org/2/library/optparse.html

首先將獲取到的命令行參數選項進行判斷和拆分以后轉變成dict鍵值對的形式存入到cmdLineOptions。然后開始依據傳入的參數進行后續操作。

在獲取命令行參數的時候，有很多dirty hack寫法，感興趣可以好好品味。這個層次的認知來源於對底層庫函數的熟悉。再次感慨，好牛！！！

這里重要的一個操作是_mergeOptions()，主要的作用是將配置項中的參數和命令行獲得的參數選項以及缺省選項進行合並。函數是這么寫的：

def _mergeOptions(inputOptions, overrideOptions):
    """
    Merge command line options with configuration file and default options.

    @param inputOptions: optparse object with command line options.
    @type inputOptions: C{instance}
    """

    if inputOptions.pickledOptions:
        try:
            unpickledOptions = base64unpickle(inputOptions.pickledOptions, unsafe=True)

            if type(unpickledOptions) == dict:
                unpickledOptions = AttribDict(unpickledOptions)

            _normalizeOptions(unpickledOptions)

            unpickledOptions["pickledOptions"] = None
            for key in inputOptions:
                if key not in unpickledOptions:
                    unpickledOptions[key] = inputOptions[key]

            inputOptions = unpickledOptions
        except Exception, ex:
            errMsg = "provided invalid value '%s' for option '--pickled-options'" % inputOptions.pickledOptions
            errMsg += " (%s)" % repr(ex)
            raise SqlmapSyntaxException(errMsg)

    if inputOptions.configFile:
        configFileParser(inputOptions.configFile)

    if hasattr(inputOptions, "items"):
        inputOptionsItems = inputOptions.items()
    else:
        inputOptionsItems = inputOptions.__dict__.items()

    for key, value in inputOptionsItems:
        if key not in conf or value not in (None, False) or overrideOptions:
            conf[key] = value

    if not hasattr(conf, "api"):
        for key, value in conf.items():
            if value is not None:
                kb.explicitSettings.add(key)

    for key, value in defaults.items():
        if hasattr(conf, key) and conf[key] is None:
            conf[key] = value

    lut = {}
    for group in optDict.keys():
        lut.update((_.upper(), _) for _ in optDict[group])

    envOptions = {}
    for key, value in os.environ.items():
        if key.upper().startswith(SQLMAP_ENVIRONMENT_PREFIX):
            _ = key[len(SQLMAP_ENVIRONMENT_PREFIX):].upper()
            if _ in lut:
                envOptions[lut[_]] = value

    if envOptions:
        _normalizeOptions(envOptions)
        for key, value in envOptions.items():
            conf[key] = value

    mergedOptions.update(conf)

然后我們來調試一下，打印一下最后的mergedOptions,結果如下：

{'code': None, 'getUsers': None, 'resultsFilename': None, 'excludeSysDbs': None, 'ignoreTimeouts': None, 'skip': None, 'db': None, 'prefix': None, 'osShell': None, 'googlePage': 1, 'query': None, 'getComments': None, 'randomAgent': None, 'testSkip': None, 'authType': None, 'getPasswordHashes': None, 'parameters': {}, 'predictOutput': None, 'wizard': None, 'stopFail': None, 'forms': None, 'uChar': None, 'authUsername': None, 'pivotColumn': None, 'dropSetCookie': None, 'dbmsCred': None, 'tests': [], 'paramExclude': None, 'risk': 1, 'sqlFile': None, 'rParam': None, 'getCurrentUser': None, 'notString': None, 'getRoles': None, 'getPrivileges': None, 'testParameter': None, 'tbl': None, 'offline': None, 'trafficFile': None, 'osSmb': None, 'level': 1, 'dnsDomain': None, 'skipStatic': None, 'secondOrder': None, 'hashDBFile': None, 'method': None, 'skipWaf': None, 'osBof': None, 'hostname': None, 'firstChar': None, 'torPort': None, 'wFile': None, 'binaryFields': None, 'checkTor': None, 'commonTables': None, 'direct': None, 'paramDict': {}, 'proxyList': None, 'titles': None, 'getSchema': None, 'timeSec': 5, 'paramDel': None, 'safeReqFile': None, 'port': None, 'getColumns': None, 'headers': None, 'crawlExclude': None, 'authCred': None, 'boundaries': [], 'loadCookies': None, 'showVersion': None, 'outputDir': None, 'tmpDir': None, 'disablePrecon': None, 'murphyRate': None, 'invalidLogical': None, 'getCurrentDb': None, 'hexConvert': None, 'proxyFile': None, 'answers': None, 'resultsFP': None, 'host': None, 'dependencies': None, 'cookie': None, 'dbmsHandler': None, 'path': None, 'alert': None, 'optimize': None, 'safeUrl': None, 'limitStop': None, 'search': None, 'uFrom': None, 'requestFile': None, 'noCast': None, 'testFilter': None, 'eta': None, 'dumpPath': None, 'csrfToken': None, 'threads': 1, 'logFile': None, 'os': None, 'col': None, 'proxy': None, 'proxyCred': None, 'verbose': 1, 'crawlDepth': None, 'updateAll': None, 'privEsc': None, 'forceDns': None, 'getAll': None, 'cj': None, 'hpp': None, 'tmpPath': None, 'header': None, 'url': u'www.baidu.com', 'invalidBignum': None, 'regexp': None, 'getDbs': None, 'httpHeaders': [], 'outputPath': None, 'freshQueries': None, 'uCols': None, 'smokeTest': None, 'ignoreProxy': None, 'regData': None, 'udfInject': None, 'invalidString': None, 'tor': None, 'forceSSL': None, 'ignore401': None, 'beep': None, 'noEscape': None, 'configFile': None, 'ipv6': False, 'scope': None, 'scheme': None, 'authFile': None, 'dbmsConnector': None, 'torType': 'SOCKS5', 'regVal': None, 'string': None, 'hashDB': None, 'mnemonics': None, 'skipUrlEncode': None, 'referer': None, 'agent': None, 'regType': None, 'purgeOutput': None, 'retries': 3, 'wFileType': None, 'extensiveFp': None, 'dumpTable': None, 'advancedHelp': None, 'batch': None, 'limitStart': None, 'flushSession': None, 'osCmd': None, 'suffix': None, 'smart': None, 'regDel': None, 'shLib': None, 'sitemapUrl': None, 'identifyWaf': None, 'msfPath': None, 'dumpAll': None, 'getHostname': None, 'sessionFile': None, 'delay': 0, 'disableColoring': None, 'getTables': None, 'safeFreq': None, 'liveTest': None, 'multipleTargets': False, 'lastChar': None, 'authPassword': None, 'nullConnection': None, 'dbms': None, 'forceThreads': None, 'dumpWhere': None, 'tamper': None, 'ignoreRedirects': None, 'charset': None, 'runCase': None, 'regKey': None, 'osPwn': None, 'evalCode': None, 'cleanup': None, 'csrfUrl': None, 'isDba': None, 'getBanner': None, 'profile': None, 'regRead': None, 'bulkFile': None, 'csvDel': ',', 'excludeCol': None, 'dumpFormat': 'CSV', 'safePost': None, 'rFile': None, 'user': None, 'parseErrors': None, 'getCount': None, 'dFile': None, 'data': None, 'regAdd': None, 'dummy': None, 'trafficFP': None, 'dnsServer': None, 'sqlmapShell': None, 'mobile': None, 'googleDork': None, 'timeout': 30, 'pickledOptions': None, 'saveConfig': None, 'sqlShell': None, 'pageRank': None, 'tech': 'BEUSTQ', 'textOnly': None, 'cookieDel': None, 'commonColumns': None, 'keepAlive': None}

接着對是否調用api，以及傳入的api參數進行處理。

 

0x02 命令行參數處理

 

然后就是十分核心的參數處理了，調用init()函數進行處理：

def init():
    """
    Set attributes into both configuration and knowledge base singletons
    based upon command line and configuration file options.
    """

    _useWizardInterface()
    setVerbosity()
    _saveConfig()
    _setRequestFromFile()
    _cleanupOptions()
    _cleanupEnvironment()
    _dirtyPatches()
    _purgeOutput()
    _checkDependencies()
    _createTemporaryDirectory()
    _basicOptionValidation()
    _setProxyList()
    _setTorProxySettings()
    _setDNSServer()
    _adjustLoggingFormatter()
    _setMultipleTargets()
    _setTamperingFunctions()
    _setWafFunctions()
    _setTrafficOutputFP()
    _resolveCrossReferences()
    _checkWebSocket()

    parseTargetUrl()
    parseTargetDirect()

    if any((conf.url, conf.logFile, conf.bulkFile, conf.sitemapUrl, conf.requestFile, conf.googleDork, conf.liveTest)):
        _setHTTPTimeout()
        _setHTTPExtraHeaders()
        _setHTTPCookies()
        _setHTTPReferer()
        _setHTTPHost()
        _setHTTPUserAgent()
        _setHTTPAuthentication()
        _setHTTPHandlers()
        _setDNSCache()
        _setSocketPreConnect()
        _setSafeVisit()
        _doSearch()
        _setBulkMultipleTargets()
        _setSitemapTargets()
        _checkTor()
        _setCrawler()
        _findPageForms()
        _setDBMS()
        _setTechnique()

    _setThreads()
    _setOS()
    _setWriteFile()
    _setMetasploit()
    _setDBMSAuthentication()
    loadBoundaries()
    loadPayloads()
    _setPrefixSuffix()
    update()
    _loadQueries()

_useWizardInterface()這個函數是為了給初學者提供一個友好的界面。

運行如下：

交互式的輸入參數。這樣你可以跟着這個提示來進行注入測試。

然后是setVerbosity()主要目的是設置了報錯的等級，函數如下：

def setVerbosity():
    """
    This function set the verbosity of sqlmap output messages.
    """

    if conf.verbose is None:
        conf.verbose = 1

    conf.verbose = int(conf.verbose)

    if conf.verbose == 0:
        logger.setLevel(logging.ERROR)
    elif conf.verbose == 1:
        logger.setLevel(logging.INFO)
    elif conf.verbose > 2 and conf.eta:
        conf.verbose = 2
        logger.setLevel(logging.DEBUG)
    elif conf.verbose == 2:
        logger.setLevel(logging.DEBUG)
    elif conf.verbose == 3:
        logger.setLevel(CUSTOM_LOGGING.PAYLOAD)
    elif conf.verbose == 4:
        logger.setLevel(CUSTOM_LOGGING.TRAFFIC_OUT)
    elif conf.verbose >= 5:
        logger.setLevel(CUSTOM_LOGGING.TRAFFIC_IN)

_saveConfig()的作用是將程序運行過程中的配置選項保存到一個文件中。命令使用是這樣的：

python ./sqlmap.py -u "http://vultest.com/index.php?id=1" --save save.txt

然后會生成一個文件進行保存。生成的東西就不貼了。整個東西夠大的了，以免影響閱讀體驗。

接下來是_setRequestFromFile()函數，整個函數的作用是處理-r 選項，也就是處理傳入一個文件。這個文件可以是burpsuite抓取到的數據包。使用方法如下：

python ./sqlmap.py -r test.txt

這里有個python的小bug，如果傳入的是'~'，將會出現問題。bug地址：http://bugs.python.org/issue18171

為了處理這個問題，我們用os模塊中路徑處理方法os.path.expanduser()進行處理。它的作用是將'~'替換成用戶目錄。運行如下：

>>> os.path.expanduser('~')
'C:\\Users\\10920'
>>>

然后調用_parseBurpLog()方法來處理burp抓取到的數據包，實現如下：

 

def _parseBurpLog(content):
        """
        Parses burp logs
        """

        if not re.search(BURP_REQUEST_REGEX, content, re.I | re.S):
            if re.search(BURP_XML_HISTORY_REGEX, content, re.I | re.S):
                reqResList = []
                for match in re.finditer(BURP_XML_HISTORY_REGEX, content, re.I | re.S):
                    port, request = match.groups()
                    try:
                        request = request.decode("base64")
                    except binascii.Error:
                        continue
                    _ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
                    if _:
                        host = _.group(0).strip()
                        if not re.search(r":\d+\Z", host):
                            request = request.replace(host, "%s:%d" % (host, int(port)))
                    reqResList.append(request)
            else:
                reqResList = [content]
        else:
            reqResList = re.finditer(BURP_REQUEST_REGEX, content, re.I | re.S)

        for match in reqResList:
            request = match if isinstance(match, basestring) else match.group(0)
            request = re.sub(r"\A[^\w]+", "", request)

            schemePort = re.search(r"(http[\w]*)\:\/\/.*?\:([\d]+).+?={10,}", request, re.I | re.S)

            if schemePort:
                scheme = schemePort.group(1)
                port = schemePort.group(2)
            else:
                scheme, port = None, None

            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
                continue

            if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
                continue

            getPostReq = False
            url = None
            host = None
            method = None
            data = None
            cookie = None
            params = False
            newline = None
            lines = request.split('\n')
            headers = []

            for index in xrange(len(lines)):
                line = lines[index]

                if not line.strip() and index == len(lines) - 1:
                    break

                newline = "\r\n" if line.endswith('\r') else '\n'
                line = line.strip('\r')
                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None

                if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                    data = ""
                    params = True

                elif match:
                    method = match.group(1)
                    url = match.group(2)

                    if any(_ in line for _ in ('?', '=', CUSTOM_INJECTION_MARK_CHAR)):
                        params = True

                    getPostReq = True

                # POST parameters
                elif data is not None and params:
                    data += "%s%s" % (line, newline)

                # GET parameters
                elif "?" in line and "=" in line and ": " not in line:
                    params = True

                # Headers
                elif re.search(r"\A\S+:", line):
                    key, value = line.split(":", 1)
                    value = value.strip().replace("\r", "").replace("\n", "")

                    # Cookie and Host headers
                    if key.upper() == HTTP_HEADER.COOKIE.upper():
                        cookie = value
                    elif key.upper() == HTTP_HEADER.HOST.upper():
                        if '://' in value:
                            scheme, value = value.split('://')[:2]
                        splitValue = value.split(":")
                        host = splitValue[0]

                        if len(splitValue) > 1:
                            port = filterStringValue(splitValue[1], "[0-9]")

                    # Avoid to add a static content length header to
                    # headers and consider the following lines as
                    # POSTed data
                    if key.upper() == HTTP_HEADER.CONTENT_LENGTH.upper():
                        params = True

                    # Avoid proxy and connection type related headers
                    elif key not in (HTTP_HEADER.PROXY_CONNECTION, HTTP_HEADER.CONNECTION):
                        headers.append((getUnicode(key), getUnicode(value)))

                    if CUSTOM_INJECTION_MARK_CHAR in re.sub(PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "", value or ""):
                        params = True

            data = data.rstrip("\r\n") if data else data

            if getPostReq and (params or cookie):
                if not port and isinstance(scheme, basestring) and scheme.lower() == "https":
                    port = "443"
                elif not scheme and port == "443":
                    scheme = "https"

                if conf.forceSSL:
                    scheme = "https"
                    port = port or "443"

                if not host:
                    errMsg = "invalid format of a request file"
                    raise SqlmapSyntaxException, errMsg

                if not url.startswith("http"):
                    url = "%s://%s:%s%s" % (scheme or "http", host, port or "80", url)
                    scheme = None
                    port = None

                if not(conf.scope and not re.search(conf.scope, url, re.I)):
                    if not kb.targets or url not in addedTargetUrls:
                        kb.targets.add((url, conf.method or method, data, cookie, tuple(headers)))
                        addedTargetUrls.add(url)

 然后_cleanupOptions()函數和_cleanupEnvironment()函數的主要作用是對傳入的數據進行清理，比如去除多余空格，對URL進行處理等。

中間的就不一一展開來說了，基本上是創建臨時目錄，如果沒有指定那么將會去創建。還有就是對傳入的參數進行處理，比如說判斷是否是int類型等等。是否符合預期規范。

然后_setProxyList()函數的作用是設置代理，並且從傳入的文件中獲取代理，如果沒有傳值將會直接return。

_setTorProxySettings()方法顧名思義就是洋蔥代理了。_setMultipleTargets()方法是處理多任務掃描。發包的任務依舊是交給_feedTargetsDict()函數。后邊我們將會詳細講解這個函數是如何進行處理的。

_setTamperingFunctions()將會根據用戶傳入的--tamper參數從目錄去調用相應的文件並且對注入時候的所需要的payload進行編碼等處理。

_setWafFunctions()方法根據參數去判斷是否有waf.

然后是兩個對URL進行分析並且判斷是否有跳轉的的函數：parseTargetUrl() 和 parseTargetDirect() 源碼如下：

def parseTargetUrl():
    """
    Parse target URL and set some attributes into the configuration singleton.
    """

    if not conf.url:
        return

    originalUrl = conf.url

    if re.search("\[.+\]", conf.url) and not socket.has_ipv6:
        errMsg = "IPv6 addressing is not supported "
        errMsg += "on this platform"
        raise SqlmapGenericException(errMsg)

    if not re.search("^http[s]*://", conf.url, re.I) and \
            not re.search("^ws[s]*://", conf.url, re.I):
        if ":443/" in conf.url:
            conf.url = "https://" + conf.url
        else:
            conf.url = "http://" + conf.url

    if CUSTOM_INJECTION_MARK_CHAR in conf.url:
        conf.url = conf.url.replace('?', URI_QUESTION_MARKER)

    try:
        urlSplit = urlparse.urlsplit(conf.url)
    except ValueError, ex:
        errMsg = "invalid URL '%s' has been given ('%s'). " % (conf.url, getSafeExString(ex))
        errMsg += "Please be sure that you don't have any leftover characters (e.g. '[' or ']') "
        errMsg += "in the hostname part"
        raise SqlmapGenericException(errMsg)

    hostnamePort = urlSplit.netloc.split(":") if not re.search("\[.+\]", urlSplit.netloc) else filter(None, (re.search("\[.+\]", urlSplit.netloc).group(0), re.search("\](:(?P<port>\d+))?", urlSplit.netloc).group("port")))

    conf.scheme = urlSplit.scheme.strip().lower() if not conf.forceSSL else "https"
    conf.path = urlSplit.path.strip()
    conf.hostname = hostnamePort[0].strip()

    conf.ipv6 = conf.hostname != conf.hostname.strip("[]")
    conf.hostname = conf.hostname.strip("[]").replace(CUSTOM_INJECTION_MARK_CHAR, "")

    try:
        _ = conf.hostname.encode("idna")
    except LookupError:
        _ = conf.hostname.encode(UNICODE_ENCODING)
    except UnicodeError:
        _ = None

    if any((_ is None, re.search(r'\s', conf.hostname), '..' in conf.hostname, conf.hostname.startswith('.'), '\n' in originalUrl)):
        errMsg = "invalid target URL ('%s')" % originalUrl
        raise SqlmapSyntaxException(errMsg)

    if len(hostnamePort) == 2:
        try:
            conf.port = int(hostnamePort[1])
        except:
            errMsg = "invalid target URL"
            raise SqlmapSyntaxException(errMsg)
    elif conf.scheme == "https":
        conf.port = 443
    else:
        conf.port = 80

    if conf.port < 0 or conf.port > 65535:
        errMsg = "invalid target URL's port (%d)" % conf.port
        raise SqlmapSyntaxException(errMsg)

    conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))
    conf.url = conf.url.replace(URI_QUESTION_MARKER, '?')

    if urlSplit.query:
        if '=' not in urlSplit.query:
            conf.url = "%s?%s" % (conf.url, getUnicode(urlSplit.query))
        else:
            conf.parameters[PLACE.GET] = urldecode(urlSplit.query) if urlSplit.query and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in urlSplit.query else urlSplit.query

    if not conf.referer and (intersect(REFERER_ALIASES, conf.testParameter, True) or conf.level >= 3):
        debugMsg = "setting the HTTP Referer header to the target URL"
        logger.debug(debugMsg)
        conf.httpHeaders = filter(lambda (key, value): key != HTTP_HEADER.REFERER, conf.httpHeaders)
        conf.httpHeaders.append((HTTP_HEADER.REFERER, conf.url.replace(CUSTOM_INJECTION_MARK_CHAR, "")))

    if not conf.host and (intersect(HOST_ALIASES, conf.testParameter, True) or conf.level >= 5):
        debugMsg = "setting the HTTP Host header to the target URL"
        logger.debug(debugMsg)
        conf.httpHeaders = filter(lambda (key, value): key != HTTP_HEADER.HOST, conf.httpHeaders)
        conf.httpHeaders.append((HTTP_HEADER.HOST, getHostHeader(conf.url)))

    if conf.url != originalUrl:
        kb.originalUrls[conf.url] = originalUrl

接下來依次根據所獲取到的參數設置請求中的延遲等待時長，設置請求頭，設置cookie信息，設置refer，DNS以及數據庫類型以及所用的注入技術等有關注入的信息。

然后再去設置線程數，加載payload等。基本的框架加載流程就是如此。后續的章節，我們將會詳細介紹有關於注入的技術，以及后滲透環節中SQLmap是怎樣做處理的。如果有有意思的代碼，那么我也會持續去做分享。