netcore實踐：跨平台動態加載native組件

緣起netcore框架下實現基於zmq的應用。

 

在.net framework時代，我們進行zmq開發由很多的選擇，比較常用的有clrzmq4和NetMQ。 其中clrzmq是基於libzmq的Interop包裝，

NetMQ是100%C#的zmq實現（基於AsyncIO組件）。以上兩種組件我都有過應用，孰優孰劣各有千秋，本文就不詳談了。

 

回歸正題，netcore下使用zmq首先也是想到引用上述組件，實踐后發現clrzmq暫時沒有基於netstandard或者netcore的支持，而NetMQ做的比較好，已經基於

netstandard1.3進行了支持。

一番折騰，搭程序，配環境。。。而后 dotnet run ，在windows下正常呈現了zmq的各項功能。

 

於是繼續dotnet publlish，通過Wnscp拷貝到centos7下執行。立即報錯，

挺意外的，本以為NetMQ已經基於netstandard進行了支持，也應該對跨平台進行支持。 可事實是AsyncIO的IOControl方法not supported on linux platform/

 

無奈，網上搜了會，也沒找到任何關於netcore在linux下進行zmq的相關內容， 事實上也沒有看到AsyncIO或者NetMQ有關於跨平台支持的說明。

既然現有方式行不通那就只好自己動手了，自己操刀通過Interop包裝libzmq來實現跨平台的zmq應用吧！

 

首先看下libzmq的組件目錄： 按x86和x64平台分為i386文件夾和amd64文件夾，且都包含windos下的dll組件和linux下的so組件

這挺難辦了，要想做好還得考慮平台類型 和 操作系統類型，  還要想想 netcore里有沒有相關API提供。 

 

先是網上搜了圈，也極少有關於netcore進行平台判斷相關內容。根據以往在framework下的經驗，直接到https://apisof.net搜索相關關鍵字

：OSPlatform

非常不錯，netcore已經提供了，同理搜索了 OSArchitecture 、DllImport 等都發現netcore1.1版本已經實現了相關api （說白了是netstandard已經實現了相關API）

 

准備就緒，直接開干,首先是平台相關信息判斷，並加載目標組件 （具體方式請參照如下代碼）

   class ZmqNative
    {
        private const string LibraryName = "libzmq";

        const int RTLD_NOW = 2; // for dlopen's flags
        const int RTLD_GLOBAL = 8;
        [DllImport(@"libdl")]
        static extern IntPtr dlopen(string filename, int flags);
        [DllImport("libdl")]
        protected static extern IntPtr dlsym(IntPtr handle, string symbol);


        [DllImport("kernel32.dll")]
        static extern IntPtr LoadLibrary(string filename);

        private static IntPtr LibPtr = IntPtr.Zero;
        static ZmqNative()
        {
           
            Console.WriteLine("OSArchitecture:{0}",RuntimeInformation.OSArchitecture);
            try {
                var libPath = @"i386";
                if (RuntimeInformation.OSArchitecture == Architecture.X86)
                {
                    libPath = @"i386";
                }
                else if (RuntimeInformation.OSArchitecture == Architecture.X64)
                {
                    libPath = @"amd64";
                }
                else
                {
                    Console.WriteLine("OSArchitecture not suported!");
                }

                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                {
                    var libName = $"{AppContext.BaseDirectory}\\{libPath}\\{LibraryName}.dll";
                    Console.WriteLine("windows:{0}", libName);
                    LibPtr = LoadLibrary(libName);
                   
                }
                else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
                {
                    var libName = $"{AppContext.BaseDirectory}/{libPath}/{LibraryName}.so";
                    Console.WriteLine("linux:{0}", libName);
                    LibPtr = dlopen(libName, RTLD_NOW|RTLD_GLOBAL);

                    
                    if(LibPtr!=IntPtr.Zero)
                    {
                        var ptr1 = dlsym(LibPtr, "zmq_ctx_new");
                        context = Marshal.GetDelegateForFunctionPointer<ZmqContext>(ptr1) ;

                        var ptr2 = dlsym(LibPtr, "zmq_socket");
                        socket = Marshal.GetDelegateForFunctionPointer<ZmqSocket>(ptr2);

                        var ptr3 = dlsym(LibPtr, "zmq_connect");
                        connect = Marshal.GetDelegateForFunctionPointer<ZmqConnect>(ptr3);

                    }
    }
                else
                {
                    Console.WriteLine("OSPlatform not suported!");
                }
                if (LibPtr != IntPtr.Zero)
                    Console.WriteLine("load zmqlib success!");
            }
            catch(Exception ex)
            {
                Console.WriteLine("load zmqlib error:\r\n{0}",ex);
            }
        }

       


        public delegate IntPtr ZmqContext();

        [DllImport(LibraryName, EntryPoint = "zmq_ctx_new", CallingConvention=CallingConvention.Cdecl)]
        public static extern IntPtr zmq_ctx_new();
        public static ZmqContext context = null;


        public delegate IntPtr ZmqSocket(IntPtr context, Int32 type);
        [DllImport(LibraryName, EntryPoint = "zmq_socket", CallingConvention = CallingConvention.Cdecl)]
        public static extern IntPtr zmq_socket(IntPtr context, Int32 type);
        public static ZmqSocket socket = null;


        public delegate Int32 ZmqConnect(IntPtr socket, IntPtr endpoint);
        [DllImport(LibraryName, EntryPoint = "zmq_connect", CallingConvention = CallingConvention.Cdecl)]
        public static extern Int32 zmq_connect(IntPtr socket, IntPtr endpoint);
        public static ZmqConnect connect = null;


        [DllImport(LibraryName, EntryPoint = "zmq_errno", CallingConvention = CallingConvention.Cdecl)]
        public static extern Int32 zmq_errno();

        [DllImport(LibraryName, EntryPoint = "zmq_strerror", CallingConvention = CallingConvention.Cdecl)]
        public static extern IntPtr zmq_strerror(int errnum);
    }

以上為測試代碼，請自動忽略代碼質量！

 

簡單解釋下，如上代碼通過平台判斷，動態加載組件，采用LoadLibaray的方式。 有心的同學可能會發現幾個delegate並且在Linux部分內通過dlsym獲取了函數指針，具體原因下面會講。

 

以上測試代碼，在windows平台下同樣正常無誤， 而在linux下還是遇到幾個小坑~~容我慢慢道來：

1、通過DllImport進行Interop的時候，組件路徑必須是確定的，這就引起了如何動態加載不同目錄下組件的問題；

    好在windows平台下通過LoadLibaray加載dll到進程空間后，DllImport標記的函數就從進程空間查找，不會重復import組件了。

   而同樣的原理在linux下用dlopen卻不能實現，還是會提示找不到組件

 

2、初次部署centos7上時，報找不到libdl.so組件問題，主要原因是系統下沒有glibc的原因，該問題可以通過yum安裝glibc的方式解決;

  

//先查找系統內是否存在組件
$ sudo find / -name libdl*

//如不存在則安裝glibc
# yum install glibc

#安裝完畢后進行鏈接
$ sudo ln -s /usr/lib64/libdl.so.2 /usr/lib64/libdl

3、解決了libdl組件問題后，繼續運行還是會發現報找不到libzmq組件的問題，實際就產生了問題1中所描述的，在linux系統下dlopen后，Interop過的函數並不會從進程空間查找。

 

 

為了解決上面遇到的問題，我們還有一條辦法，就是創建 delegate ， 並且通過LoadLibaray組件后通過GetProcAddress方式獲取函數指針了。  具體的解決方案在上述測試代碼已經體現了，這里就不過多解釋了。

 

以上，就全部解決了在 netcore框架基礎上進行跨平台native組件應用的問題。 真實測試結果如圖：

 

請主動忽略初zmq應用外的其他信息， 本次測試一同測試了通過App入口啟動webapi +  websockets + zmq ，api創建為aspnetcore在Web.dll內，websockets在Lib.dll內，zmq在App.dll內。

 

補充下完整測試代碼：

using System;
using System.Collections.Generic;
using System.Text;
using System.Runtime.InteropServices;

namespace App
{
    
    class ZmqNative
    {
        private const string LibraryName = "libzmq";

        const int RTLD_NOW = 2; // for dlopen's flags
        const int RTLD_GLOBAL = 8;
        [DllImport(@"libdl")]
        static extern IntPtr dlopen(string filename, int flags);
        [DllImport("libdl")]
        protected static extern IntPtr dlsym(IntPtr handle, string symbol);


        [DllImport("kernel32.dll")]
        static extern IntPtr LoadLibrary(string filename);

        private static IntPtr LibPtr = IntPtr.Zero;
        static ZmqNative()
        {
           
            Console.WriteLine("OSArchitecture:{0}",RuntimeInformation.OSArchitecture);
            try {
                var libPath = @"i386";
                if (RuntimeInformation.OSArchitecture == Architecture.X86)
                {
                    libPath = @"i386";
                }
                else if (RuntimeInformation.OSArchitecture == Architecture.X64)
                {
                    libPath = @"amd64";
                }
                else
                {
                    Console.WriteLine("OSArchitecture not suported!");
                }

                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                {
                    var libName = $"{AppContext.BaseDirectory}\\{libPath}\\{LibraryName}.dll";
                    Console.WriteLine("windows:{0}", libName);
                    LibPtr = LoadLibrary(libName);
                   
                }
                else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
                {
                    var libName = $"{AppContext.BaseDirectory}/{libPath}/{LibraryName}.so";
                    Console.WriteLine("linux:{0}", libName);
                    LibPtr = dlopen(libName, RTLD_NOW|RTLD_GLOBAL);

                    
                    if(LibPtr!=IntPtr.Zero)
                    {
                        var ptr1 = dlsym(LibPtr, "zmq_ctx_new");
                        context = Marshal.GetDelegateForFunctionPointer<ZmqContext>(ptr1) ;

                        var ptr2 = dlsym(LibPtr, "zmq_socket");
                        socket = Marshal.GetDelegateForFunctionPointer<ZmqSocket>(ptr2);

                        var ptr3 = dlsym(LibPtr, "zmq_connect");
                        connect = Marshal.GetDelegateForFunctionPointer<ZmqConnect>(ptr3);

                    }
    }
                else
                {
                    Console.WriteLine("OSPlatform not suported!");
                }
                if (LibPtr != IntPtr.Zero)
                    Console.WriteLine("load zmqlib success!");
            }
            catch(Exception ex)
            {
                Console.WriteLine("load zmqlib error:\r\n{0}",ex);
            }
        }

       


        public delegate IntPtr ZmqContext();

        [DllImport(LibraryName, EntryPoint = "zmq_ctx_new", CallingConvention=CallingConvention.Cdecl)]
        public static extern IntPtr zmq_ctx_new();
        public static ZmqContext context = null;


        public delegate IntPtr ZmqSocket(IntPtr context, Int32 type);
        [DllImport(LibraryName, EntryPoint = "zmq_socket", CallingConvention = CallingConvention.Cdecl)]
        public static extern IntPtr zmq_socket(IntPtr context, Int32 type);
        public static ZmqSocket socket = null;


        public delegate Int32 ZmqConnect(IntPtr socket, IntPtr endpoint);
        [DllImport(LibraryName, EntryPoint = "zmq_connect", CallingConvention = CallingConvention.Cdecl)]
        public static extern Int32 zmq_connect(IntPtr socket, IntPtr endpoint);
        public static ZmqConnect connect = null;


        [DllImport(LibraryName, EntryPoint = "zmq_errno", CallingConvention = CallingConvention.Cdecl)]
        public static extern Int32 zmq_errno();

        [DllImport(LibraryName, EntryPoint = "zmq_strerror", CallingConvention = CallingConvention.Cdecl)]
        public static extern IntPtr zmq_strerror(int errnum);
    }

    class ZContext
    {
        private static IntPtr _context = IntPtr.Zero;
        static ZContext()
        {
            if (ZmqNative.context != null)
            {
                _context = ZmqNative.context();
            }
            else
            {
                _context = ZmqNative.zmq_ctx_new();
            }
            
            if (_context == IntPtr.Zero)
            {
                Console.WriteLine("zerror:{0}", ZError.GetLastError());
            }
        }

        public static IntPtr Current
        {
            get
            {
                return _context;
            }
        }
    }


    class ZError
    {
        public static string GetLastError()
        {
            var error = string.Empty;
            var no = ZmqNative.zmq_errno();
            if (no != 0)
            {
                var ptr = ZmqNative.zmq_strerror(no);
                error = Marshal.PtrToStringUTF8(ptr);
            }
            return error;
        }
    }

    enum ZmqSocketType :int {
        SUB = 2
    }

    class ClrZmq
    {
        private IntPtr zmq = IntPtr.Zero;
        public ClrZmq(string url) {
            Url = url;
        }

        public string Url { get; private set; }

        private bool _IsStarted= false;
        public bool IsStarted
        {
            get
            {
                return _IsStarted;
            }
        }

        public void Start() {
            if (IsStarted) return;
            try
            {
                if (ZmqNative.socket != null)
                    zmq = ZmqNative.socket(ZContext.Current, (int)ZmqSocketType.SUB);
                else
                    zmq = ZmqNative.zmq_socket(ZContext.Current, (int)ZmqSocketType.SUB);
                if (zmq == IntPtr.Zero)
                {
                    //error
                    Console.WriteLine("zerror:{0}",ZError.GetLastError());
                    return;
                }
                Console.WriteLine("create mq success!");
                
                var str =  Marshal.StringToHGlobalAnsi(Url);

                var ret = 0;
                if (ZmqNative.connect != null)
                    ret = ZmqNative.connect(zmq, str);
                else
                    ret = ZmqNative.zmq_connect(zmq, str);
                if ( ret!= 0)
                {
                    //error
                    Console.WriteLine("zerror:{0}",ZError.GetLastError());
                    return;
                }
                Marshal.FreeHGlobal(str);

                Console.WriteLine("connect zmq success!");

            }
            catch (Exception ex) {
                Console.WriteLine(ex);
            }
        }
    }
}