Miscellaneous Notes: JsonCpp

續http://zf-php-oo.blogspot.tw/2014/11/c11phpcombinejsonphp-proxy-classphp.html此編，由於該篇是擷取system()的response值，光是開兩個thread去擷取response就花了90~200ms。因為太慢的關係，之前有嘗試調用libcurl裡的curl.h去擷取http連線的response，一樣開兩個連線，只加快到78~90ms。個人猜想libcurl是用C寫的，會是因為用C++調用C開發出來的libcurl，所以沒快到哪裡去嗎?後來用了Poco這個用C++編寫的library後，開兩個thread並擷取，個人遇到最快的反應時間是700us(微秒, 10的負6次方)，若查詢的sql裡的where條件不包含任何key，大約在14ms以內。移植到gentoo with Xeon Ivy Bridge Based 2.2G(6核/12thread)上，開5個thread去擷取，大約在25~35ms以內。

multiThreadQuery.h:

#ifndef MULTITHREADQUERY_H
#define MULTITHREADQUERY_H
#endif

#include "/usr/include/jsoncpp/json/json.h"

#include <string>

class multiThreadQuery{
public:
    struct inputOutput{
        std::string url;
        std::string alias;
        std::string response;
    };

    explicit multiThreadQuery();

    ~multiThreadQuery();


    // 為配合SWIG不支援return reference，故於此return by pointer
    multiThreadQuery* clearAllUrls();

    multiThreadQuery* attachUrl(std::string url, std::string alias = "");

    multiThreadQuery* fetchResponse(bool yesNo);

    std::string exe();
protected:
    int _threadNum;
    Json::Value _urlArr;
    bool _fetchResponse;

    void _curl(inputOutput& io);
};

multiThreadQuery.cpp:

#include "multiThreadQuery.h"
#include "/usr/include/Poco/Net/HTTPClientSession.h"
#include "/usr/include/Poco/Net/HTTPRequest.h"
#include "/usr/include/Poco/Net/HTTPResponse.h"
#include "/usr/include/Poco/StreamCopier.h"
#include "/usr/include/Poco/Path.h"
#include "/usr/include/Poco/URI.h"
#include "/usr/include/Poco/Exception.h"

#include <iostream>
#include <thread>
#include <cstdlib>
#include <sstream>
#include <chrono>

using namespace std;
using namespace Poco;
using namespace Poco::Net;

multiThreadQuery::multiThreadQuery(void): _threadNum(0), _fetchResponse(false){
}

multiThreadQuery::~multiThreadQuery(void){
}

multiThreadQuery* multiThreadQuery::clearAllUrls(){
    _threadNum = 0;
    Json::Value jsonValue;
    _urlArr = jsonValue;

    return this;
}

multiThreadQuery* multiThreadQuery::attachUrl(std::string url, std::string alias){
    // 整數轉字串
    std::string str;
    stringstream ss(str);
    ss << _threadNum;

    _urlArr[ss.str()]["url"] = url;

    if(alias != ""){
        _urlArr[ss.str()]["alias"] = alias;
    }else{
        // time_point的time_since_epoch()才會包含小數點
        chrono::system_clock::time_point tp = chrono::system_clock::now();
        chrono::microseconds us = chrono::duration_cast(tp.time_since_epoch());
        time_t tt = us.count();

        stringstream strm;
        strm << tt;
        _urlArr[ss.str()]["alias"] = strm.str();
    }

    ++_threadNum;

    return this;
}

void multiThreadQuery::_curl(inputOutput& io){
    try{
        URI uri(io.url);

        HTTPClientSession session(uri.getHost(), uri.getPort());

        // prepare path
        string path(uri.getPathAndQuery());
        if(path.empty()) path = "/";

        // send request
        HTTPRequest req(HTTPRequest::HTTP_GET, path, HTTPMessage::HTTP_1_1);
        session.sendRequest(req);

        if(_fetchResponse){
            // get response
            HTTPResponse res;

            // print response
            istream &is = session.receiveResponse(res);
            std::string response;
            StreamCopier::copyToString(is, response);

            io.response = "\"" + io.alias + "\":" + response;
        }
    }catch(Poco::Exception &ex){
        io.response = "\"" + io.alias + "\":{\"s\":-5,\"desc\":\"Curl '" + io.url + "' failed.\"}";
    }
}

multiThreadQuery* multiThreadQuery::fetchResponse(bool yesNo){
    _fetchResponse = yesNo;

    return this;
}

std::string multiThreadQuery::exe(){
    int i;
    std::string json = "";
    // 動態設定array大小
    thread* threadArrPtr = new thread[_threadNum];
    inputOutput* ioPtr = new inputOutput[_threadNum];

    for(i = 0; i < _threadNum; ++i){
        // 整數轉字串
        std::string url;
        stringstream urlStream(url);
        urlStream << i;

        (*(ioPtr + i)).alias = _urlArr[urlStream.str()]["alias"].asString();
        (*(ioPtr + i)).url = _urlArr[urlStream.str()]["url"].asString();
        (*(ioPtr + i)).response = "";

        *(threadArrPtr + i) = thread(&multiThreadQuery::_curl, this, ref((*(ioPtr + i))));
    }

    if(_fetchResponse) json = "{";

    for(i = 0; i < _threadNum; ++i){
        // 不得放在產生thread的迴圈內，多執行緒會失效，變成跑一般迴圈
        (*(threadArrPtr + i)).join();

        if(_fetchResponse){
            json += (*(ioPtr + i)).response;
            if(i != _threadNum - 1) json += ",";
        }
    }

    if(_fetchResponse) json += "}";

    delete [] threadArrPtr;
    delete [] ioPtr;
    threadArrPtr = nullptr;
    ioPtr = nullptr;

    return json;
}

int main(){
    const auto start = chrono::system_clock::now();

    multiThreadQuery mtq;
    std::string response;

    response = mtq.clearAllUrls()->fetchResponse(true)
                                    ->attachUrl("http://127.0.0.1/test.php?s=1")
                                    ->attachUrl("http://127.0.0.1/test.php?s=2", "member")
                                    ->attachUrl("http://127.0.0.1/test.php?s=3", "login_log")
                                    ->attachUrl("http://127.0.0.1/test.php?s=4", "payment")
                                    ->attachUrl("http://127.0.0.1/test.php?s=5", "detail")
                                    ->exe();

    cout << response << endl;

    const auto end = chrono::system_clock::now();
    auto ts = end - start;
    cout << "spent: " << chrono::duration_cast(ts).count() << "us" << endl;

    return 0;
}

以上在linux上建入指令:
#g++ -ljsoncpp -lpthread -lPocoNet -lPocoFoundation -std=c++11 multiThreadQuery.cpp -o multiThreadQuery.out
產生binary檔後再執行:
#./multiThreadQuery.out
若執行時間無太大問題，則覆製multiThreadQuery.cpp另存一份swig的interface檔: multiThreadQuery.i，並將main()全部移除。(記得.h, .cpp, .i裡的string都得寫成std::string)

multiThreadQuery.i:

%module multiThreadQuery
%include "std_string.i"
%{
multiThreadQuery.cpp裡的內容，除了main()以外!!!
%}

class multiThreadQuery{
public:
    multiThreadQuery();
    ~multiThreadQuery();
    multiThreadQuery* clearAllUrls();
    multiThreadQuery* attachUrl(std::string url, std::string alias = "");
    multiThreadQuery* fetchResponse(bool yesNo);
    std::string exe();
};

包成PHP Extension指令:
#swig -c++ -php5 multiThreadQuery.i

#g++ `php-config --includes` -O2 -march=native -mtune=native -std=c++11 -ljsoncpp -lpthread -lPocoNet -lPocoFoundation -fPIC -c multiThreadQuery_wrap.cpp

#g++ -ljsoncpp -lpthread -lPocoNet -lPocoFoundation -shared multiThreadQuery_wrap.o -o multiThreadQuery.so

最後在php.ini檔裡載入multiThreadQuery.so且PHP script裡記得載入multiThreadQuery.php這支PHP Proxy class即可宣告並使用了。

Ex:

$mtq = new multiThreadQuery();

$response = $mtq->clearAllUrls()
                             ->fetchResponse(true)
                             ->attachUrl('http://127.0.0.1/test1.php', 'test1')
                             ->attachUrl('http://127.0.0.1/test2.php', 'test2')
                             ->exe();

前言:

我們都知道PHP是直譯式語言。如果一支php script裡執行了三斷查詢或寫入DB的code，其中第一個查詢花了一秒，那麼這支script完成總共所要花的時間就是一秒以上起跳。

目標:

用C++寫出一支class可供PHP宣告後，將各script及argument一一代入，最後產生對應的執行緒將各個script的return值集合起來，combine在一個json格式的字串並return。

PHP ex:
$mtq = new multiThreadQuery();

$response = $mtq->attach('/www/abc.php 123')
                             ->attach('/www/def.php 0 456')
                             ->attach('/www/ghi.php 1 m')
                             ->exe();

$arr = json_decode($response, true);

JsonCpp的下載點& for Visual Studio 2012的編譯方法可參考此編:
http://xyz.cinc.biz/2013/05/c-json-jsoncpp.html

以下的code都是for linux的...
除了jsconcpp路徑需修改外，system()指令可以先mark起來，其餘應該都可以在Visual Studio 2012上編譯&執行。

以下會用到C++11的struct, pointer, reference, dynamic memory allocation for array & struct, thread, 有不熟的再自己去查資料吧~P.S. 如何在linux上安裝jsoncpp, swig，就也自己上網找吧~

multiThreadQuery.h:

#ifndef MULTITHREADQUERY_H
#define MULTITHREADQUERY_H
#endif

#include <string>
#include "YOUR LINUX PATH/json/son.h"

class multiThreadQuery{
public:
   struct dataInterchange{
       std::string scriptPath;
       std::string argv;
       std::string response;
   };

   explicit multiThreadQuery();

   ~multiThreadQuery();

    // 為配合SWIG不支援return reference，故於此return by pointer
    multiThreadQuery* clearAllScript();

   multiThreadQuery* attachPHP(std::string cmd);

   std::string exe();
protected:
   int _phpThreadNum;
   Json::Value _phpPathArr;
   Json::Value _phpArgvArr;

   void _exePHP(dataInterchange& di);
};

multiThreadQuery.cpp:

#include "multiThreadQuery.h"
#include <iostream>
#include <thread>
#include <cstdlib>
#include <sstream>
#include <chrono>

using namespace std;

multiThreadQuery::multiThreadQuery() : _phpThreadNum(0){
}

multiThreadQuery::~multiThreadQuery(){
}

multiThreadQuery* multiThreadQuery::clearAllScript(){
   _phpThreadNum = 0;
   Json::Value jsonValue;
   _phpPathArr = jsonValue;
   _phpArgvArr = jsonValue;

   return this;
}
multiThreadQuery* multiThreadQuery::attachPHP(std::string cmd){
    // 整數轉字串
    std::string str;
    stringstream ss(str);
    ss << _phpThreadNum;

    _phpPathArr[ss.str()] = cmd.substr(0, cmd.find(" "));
    _phpArgvArr[ss.str()] = cmd.substr(cmd.find(" ") + 1);
    ++_phpThreadNum;

    return this;
}

std::string multiThreadQuery::exe(){
    int i;
    // 動態設定array大小
    thread* threadArrPtr = new thread[_phpThreadNum];
    dataInterchange* diPtr = new dataInterchange[_phpThreadNum];

    for(i = 0; i < _phpThreadNum; ++i){
        // 整數轉字串
        std::string str;
        stringstream ss(str);
        ss << i;

        (*(diPtr + i)).scriptPath = _phpPathArr[ss.str()].asString();
        (*(diPtr + i)).argv = _phpArgvArr[ss.str()].asString();
        (*(diPtr + i)).response = "";

        *(threadArrPtr + i) = thread(&multiThreadQuery::_exePHP, this, ref((*(diPtr + i))));
    }

    string json = "{";

    for(i = 0; i < _phpThreadNum; ++i){
        // 不得放在產生thread的迴圈內，多執行緒會失效，變成跑一般迴圈
        (*(threadArrPtr + i)).join();

        json += (*(diPtr + i)).response;
        if(i != (_phpThreadNum - 1)) json += ",";
    }

    json += "}";

    delete [] threadArrPtr;
    delete [] diPtr;
    threadArrPtr = nullptr;
    diPtr = nullptr;

    return json;
}

void multiThreadQuery::_exePHP(dataInterchange& diRef){
    try{
        if(fopen(diRef.scriptPath.c_str(), "r") == NULL) throw -999;

        FILE* fpipe;
        char* command;
        char line[256];
        std::string response;
        std::string cmd;

        cmd = "/usr/local/bin/php " + diRef.scriptPath + " " + diRef.argv;
        command = (char*)cmd.c_str();

        if (!(fpipe = (FILE*)popen(command, "r"))) throw -998;

        while(fgets(line, sizeof(line), fpipe)){
            std::string str;
            str.assign(line);
            response += str;
        }

        pclose(fpipe);

        diRef.response = "\"" + diRef.scriptPath + " " + diRef.argv + "\":" + response;
    }catch(int e){
        std::string desc;

        switch(e){
            case -999:
                desc = "File not exist.";
                break;
            case -998:
                desc = "Pipe failed.";
                break;
        }

        // 整數轉字串
        std::string str;
        stringstream ss(str);
        ss << e;

        diRef.response = "\"" + diRef.scriptPath + " " + diRef.argv + "\":{\"s\":" + ss.str() + ",\"desc\":\"" + desc + "\"}";
    }
}

int main(){
    const auto start = std::chrono::system_clock::now();

    multiThreadQuery mtq;
    cout << mtq.attachPHP("/www/abc.php 1")->attachPHP("/www/dev.php 2")->exe() << endl;

    const auto end = std::chrono::system_clock::now();
    auto ts = end - start;
    cout << "spent: " << std::chrono::duration_cast(ts).count() << "ms" << endl;

    return 0;
}

以上在linux裡使用g++編譯並執行，若沒出任何錯誤，可以將main()整個方法可以刪掉了，因為待會使用SWIG包成PHP proxy class及extension(.so)檔時都不會用到。P.S. PHP自己寫幾支吐些json_encode()的字串回來，每支可以設sleep()不同秒數，看看執行總共花的秒數是不是趨近於最花時間的script執行時間。

因為有用到jsoncpp函式庫及C++11的thread，所以編譯時待指定lpthread及ljsoncpp參數，如:
g++ -std=c++11 -ljsoncpp -lpthread multiThreadQuery.cpp -o multiThreadQuery.out

砍掉main()方法後，#cp multiThreadQuery.cpp multiThreadQuery.i。這支.i檔就是SWIG要把C++編譯出來的binary檔wrap成.so檔及php proxy class用的。

multiThreadQuery.i:

%module multiThreadQuery
%include "std_string.i"
%{
#include "multiThreadQuery.h"
#include <iostream>
#include <thread>
#include <cstdlib>
#include <sstream>

using namespace std;

multiThreadQuery::multiThreadQuery(void) : _phpThreadNum(0){
}

multiThreadQuery::~multiThreadQuery(void){
}

multiThreadQuery* multiThreadQuery::clearAllScripts(){
    _phpThreadNum = 0;
    Json::Value jsonValue;
    _phpPathArr = jsonValue;
    _phpArgvArr = jsonValue;

    return this;
}

multiThreadQuery* multiThreadQuery::attachPHP(std::string cmd){
    // 整數轉字串
    std::string str;
    stringstream ss(str);
    ss << _phpThreadNum;

    _phpPathArr[ss.str()] = cmd.substr(0, cmd.find(" "));
    _phpArgvArr[ss.str()] = cmd.substr(cmd.find(" ") + 1);
    ++_phpThreadNum;

    return this;
}

std::string multiThreadQuery::exe(){
    int i;
    // 動態設定array大小
    thread* threadArrPtr = new thread[_phpThreadNum];
    dataInterchange* diPtr = new dataInterchange[_phpThreadNum];

    for(i = 0; i < _phpThreadNum; ++i){
        // 整數轉字串
        std::string str;
        stringstream ss(str);
        ss << i;

        (*(diPtr + i)).scriptPath = _phpPathArr[ss.str()].asString();
        (*(diPtr + i)).argv = _phpArgvArr[ss.str()].asString();
        (*(diPtr + i)).response = "";

        *(threadArrPtr + i) = thread(&multiThreadQuery::_exePHP, this, ref((*(diPtr + i))));
    }

    string json = "{";

    for(i = 0; i < _phpThreadNum; ++i){
        // 不得放在產生thread的迴圈內，多執行緒會失效，變成跑一般迴圈
        (*(threadArrPtr + i)).join();

        json += (*(diPtr + i)).response;
        if(i != (_phpThreadNum - 1)) json += ",";
    }

    json += "}";

    delete [] threadArrPtr;
    delete [] diPtr;
    threadArrPtr = nullptr;
    diPtr = nullptr;

    return json;
}

void multiThreadQuery::_exePHP(dataInterchange& diRef){
    try{
        if(fopen(diRef.scriptPath.c_str(), "r") == NULL) throw -999;

        FILE* fpipe;
        char* command;
        char line[256];
        std::string response;
        std::string cmd;

        cmd = "/usr/local/bin/php " + diRef.scriptPath + " " + diRef.argv;
        command = (char*)cmd.c_str();

        if (!(fpipe = (FILE*)popen(command, "r"))) throw -998;

        while(fgets(line, sizeof(line), fpipe)){
            std::string str;
            str.assign(line);
            response += str;
        }

        pclose(fpipe);

        diRef.response = "\"" + diRef.scriptPath + " " + diRef.argv + "\":" + response;
    }catch(int e){
        std::string desc;

        switch(e){
            case -999:
                desc = "File not exist.";
                break;
            case -998:
                desc = "Pipe failed.";
                break;
        }

        // 整數轉字串
        std::string str;
        stringstream ss(str);
        ss << e;

        diRef.response = "\"" + diRef.scriptPath + " " + diRef.argv + "\":{\"s\":" + ss.str() + ",\"desc\":\"" + desc + "\"}";
    }
}
%}

class multiThreadQuery{
public:
    multiThreadQuery();
    ~multiThreadQuery();
    multiThreadQuery clearAllScripts();
    multiThreadQuery attachPHP(std::string cmd);
    std::string exe();
};

%module到%}的用法請參考: http://www.swig.org/Doc3.0/SWIGDocumentation.html#SWIGPlus_nn6，最後一小斷是為了產生一PHP proxy class，供PHP require後，即可宣告此class。PHP proxy class請參考: http://www.swig.org/Doc3.0/SWIGDocumentation.html#Php_nn2_6

#swig -c++ -php5 multiThreadQuery.i

#g++ `php-config --includes` -O2 -march=native -mtune=native -std=c++11 -ljsoncpp -lpthread -fPIC -c multiThreadQuery_wrap.cpp

#g++ -ljsoncpp -lpthread -shared multiThreadQuery_wrap.o -o multiThreadQuery.so

最後在php.ini裡載入so檔，PHP裡require multiThreadQuery.php檔，即可在PHP script裡下:

$mtq = new multiThreadQuery();

$response = $mtq->attach('/www/abc.php 123')
                             ->attach('/www/def.php 0 456')
                             ->attach('/www/ghi.php 1 m')
                             ->exe();

$arr = json_decode($response, true);

Miscellaneous Notes

2014年11月11日星期二

使用JsonCpp, Poco library製作多執行緒擷取多筆api回傳值，並combine為一json格式，再利用SWIG包成PHP Extension

2014年11月3日星期一

使用C++11編寫一可供PHP呼叫並產生多執行緒，並將所有執行緒的結果combine於一json格式內的PHP proxy class及PHP Extension(.so)

2014年11月11日 星期二

使用JsonCpp, Poco library製作多執行緒擷取多筆api回傳值，並combine為一json格式，再利用SWIG包成PHP Extension

2014年11月3日 星期一

使用C++11編寫一可供PHP呼叫並產生多執行緒，並將所有執行緒的結果combine於一json格式內的PHP proxy class及PHP Extension(.so)

2014年11月11日星期二

2014年11月3日星期一