Added ability to fetch media from remote server (using cURL library)
authorIlya Zhuravlev <zhuravlevilya@ya.ru>
Fri, 14 Dec 2012 11:30:17 +0000 (15:30 +0400)
committerIlya Zhuravlev <zhuravlevilya@ya.ru>
Sun, 16 Dec 2012 11:20:18 +0000 (15:20 +0400)
12 files changed:
cmake/Modules/FindCURL.cmake [new file with mode: 0644]
minetest.conf.example
src/CMakeLists.txt
src/client.cpp
src/client.h
src/clientserver.h
src/cmake_config.h.in
src/config.h
src/defaultsettings.cpp
src/server.cpp
src/util/string.cpp
src/util/string.h

diff --git a/cmake/Modules/FindCURL.cmake b/cmake/Modules/FindCURL.cmake
new file mode 100644 (file)
index 0000000..ec0503f
--- /dev/null
@@ -0,0 +1,17 @@
+# - Find curl
+# Find the native CURL headers and libraries.
+#
+#  CURL_INCLUDE_DIR - where to find curl/curl.h, etc.
+#  CURL_LIBRARY    - List of libraries when using curl.
+#  CURL_FOUND        - True if curl found.
+
+# Look for the header file.
+FIND_PATH(CURL_INCLUDE_DIR NAMES curl/curl.h)
+
+# Look for the library.
+FIND_LIBRARY(CURL_LIBRARY NAMES curl)
+
+# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARY CURL_INCLUDE_DIR)
index a0aa691258a2d6741eee11d4bc0d67936047921d..e80991936fa6ab037cbf78a1ecb34e2e58f8056c 100644 (file)
 # 2: enable high level shaders
 #enable_shaders = 2
 
+# will only work for servers which use remote_media setting
+# and only for clients compiled with cURL
+#media_fetch_threads = 8
+
 #
 # Server stuff
 #
 #congestion_control_aim_rtt = 0.2
 #congestion_control_max_rate = 400
 #congestion_control_min_rate = 10
+#remote_media =
index 3830ef3b66636a862690bbe0a7eb836b83ee8b81..38410f7d2d3b50fc94b1663023a80c996673c0f2 100644 (file)
@@ -6,6 +6,19 @@ mark_as_advanced(EXECUTABLE_OUTPUT_PATH LIBRARY_OUTPUT_PATH)
 mark_as_advanced(JTHREAD_INCLUDE_DIR JTHREAD_LIBRARY)
 mark_as_advanced(SQLITE3_INCLUDE_DIR SQLITE3_LIBRARY)
 
+option(ENABLE_CURL "Enable cURL support for fetching media" 1)
+
+if (NOT ENABLE_CURL)
+       mark_as_advanced(CLEAR CURL_LIBRARY CURL_INCLUDE_DIR)
+endif(NOT ENABLE_CURL)
+
+find_package(CURL)
+set(USE_CURL 0)
+if (CURL_FOUND AND ENABLE_CURL)
+       message(STATUS "cURL support enabled")
+       set(USE_CURL 1)
+endif(CURL_FOUND AND ENABLE_CURL)
+
 # user-visible option to enable/disable gettext usage
 OPTION(ENABLE_GETTEXT "Use GetText for internationalization" 0)
 
@@ -307,6 +320,16 @@ if(BUILD_CLIENT)
                ${PLATFORM_LIBS}
                ${CLIENT_PLATFORM_LIBS}
        )
+
+       if(USE_CURL)
+               target_link_libraries(
+                       ${PROJECT_NAME}
+                       ${CURL_LIBRARY}
+               )
+               include_directories(
+                       ${CURL_INCLUDE_DIR}
+               )
+       endif(USE_CURL)
 endif(BUILD_CLIENT)
 
 if(BUILD_SERVER)
index 3463e926269683605fcf24f42fdc754cde274d45..46b53c6d725e4a4ddda2c534d389cba0f15bae26 100644 (file)
@@ -44,21 +44,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "hex.h"
 #include "IMeshCache.h"
 #include "util/serialize.h"
+#include "config.h"
+
+#if USE_CURL
+#include <curl/curl.h>
+#endif
 
 static std::string getMediaCacheDir()
 {
        return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
 }
 
-struct MediaRequest
-{
-       std::string name;
-
-       MediaRequest(const std::string &name_=""):
-               name(name_)
-       {}
-};
-
 /*
        QueuedMeshUpdate
 */
@@ -223,6 +219,45 @@ void * MeshUpdateThread::Thread()
        return NULL;
 }
 
+void * MediaFetchThread::Thread()
+{
+       ThreadStarted();
+
+       log_register_thread("MediaFetchThread");
+
+       DSTACK(__FUNCTION_NAME);
+
+       BEGIN_DEBUG_EXCEPTION_HANDLER
+
+       #if USE_CURL
+       CURL *curl;
+       CURLcode res;
+       for (core::list<MediaRequest>::Iterator i = m_file_requests.begin();
+                       i != m_file_requests.end(); i++) {
+               curl = curl_easy_init();
+               assert(curl);
+               curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str());
+               curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+               std::ostringstream stream;
+               curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data);
+               curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
+               res = curl_easy_perform(curl);
+               if (res == CURLE_OK) {
+                       std::string data = stream.str();
+                       m_file_data.push_back(make_pair(i->name, data));
+               } else {
+                       m_failed.push_back(*i);
+                       infostream << "cURL request failed for " << i->name << std::endl;
+               }
+               curl_easy_cleanup(curl);
+       }
+       #endif
+
+       END_DEBUG_EXCEPTION_HANDLER(errorstream)
+
+       return NULL;
+}
+
 Client::Client(
                IrrlichtDevice *device,
                const char *playername,
@@ -263,8 +298,9 @@ Client::Client(
        m_password(password),
        m_access_denied(false),
        m_media_cache(getMediaCacheDir()),
-       m_media_receive_progress(0),
-       m_media_received(false),
+       m_media_receive_started(false),
+       m_media_count(0),
+       m_media_received_count(0),
        m_itemdef_received(false),
        m_nodedef_received(false),
        m_time_of_day_set(false),
@@ -730,6 +766,63 @@ void Client::step(float dtime)
                        g_profiler->graphAdd("num_processed_meshes", num_processed_meshes);
        }
 
+       /*
+               Load fetched media
+       */
+       if (m_media_receive_started) {
+               bool all_stopped = true;
+               for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
+                               thread != m_media_fetch_threads.end(); thread++) {
+                       all_stopped &= !thread->IsRunning();
+                       while (thread->m_file_data.size() > 0) {
+                               std::pair <std::string, std::string> out = thread->m_file_data.pop_front();
+                               ++m_media_received_count;
+
+                               bool success = loadMedia(out.second, out.first);
+                               if(success){
+                                       verbosestream<<"Client: Loaded received media: "
+                                                       <<"\""<<out.first<<"\". Caching."<<std::endl;
+                               } else{
+                                       infostream<<"Client: Failed to load received media: "
+                                                       <<"\""<<out.first<<"\". Not caching."<<std::endl;
+                                       continue;
+                               }
+
+                               bool did = fs::CreateAllDirs(getMediaCacheDir());
+                               if(!did){
+                                       errorstream<<"Could not create media cache directory"
+                                                       <<std::endl;
+                               }
+
+                               {
+                                       core::map<std::string, std::string>::Node *n;
+                                       n = m_media_name_sha1_map.find(out.first);
+                                       if(n == NULL)
+                                               errorstream<<"The server sent a file that has not "
+                                                               <<"been announced."<<std::endl;
+                                       else
+                                               m_media_cache.update_sha1(out.second);
+                               }
+                       }
+               }
+               if (all_stopped) {
+                       core::list<MediaRequest> fetch_failed;
+                       for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
+                                       thread != m_media_fetch_threads.end(); thread++) {
+                               for (core::list<MediaRequest>::Iterator request = thread->m_failed.begin();
+                                               request != thread->m_failed.end(); request++)
+                                       fetch_failed.push_back(*request);
+                               thread->m_failed.clear();
+                       }
+                       if (fetch_failed.size() > 0) {
+                               infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. "
+                                               << "Requesting them the usual way." << std::endl;
+                               request_media(fetch_failed);
+                       }
+                       m_media_fetch_threads.clear();
+               }
+       }
+
        /*
                If the server didn't update the inventory in a while, revert
                the local inventory (so the player notices the lag problem
@@ -907,6 +1000,34 @@ void Client::deletingPeer(con::Peer *peer, bool timeout)
                        <<"(timeout="<<timeout<<")"<<std::endl;
 }
 
+/*
+       u16 command
+       u16 number of files requested
+       for each file {
+               u16 length of name
+               string name
+       }
+*/
+void Client::request_media(const core::list<MediaRequest> &file_requests)
+{
+       std::ostringstream os(std::ios_base::binary);
+       writeU16(os, TOSERVER_REQUEST_MEDIA);
+       writeU16(os, file_requests.size());
+
+       for(core::list<MediaRequest>::ConstIterator i = file_requests.begin();
+                       i != file_requests.end(); i++) {
+               os<<serializeString(i->name);
+       }
+
+       // Make data buffer
+       std::string s = os.str();
+       SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+       // Send as reliable
+       Send(0, data, true);
+       infostream<<"Client: Sending media request list to server ("
+                       <<file_requests.size()<<" files)"<<std::endl;
+}
+
 void Client::ReceiveAll()
 {
        DSTACK(__FUNCTION_NAME);
@@ -1514,37 +1635,56 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
                        file_requests.push_back(MediaRequest(name));
                }
 
-               ClientEvent event;
-               event.type = CE_TEXTURES_UPDATED;
-               m_client_event_queue.push_back(event);
+               std::string remote_media = "";
+               try {
+                       remote_media = deSerializeString(is);
+               }
+               catch(SerializationError) {
+                       // not supported by server or turned off
+               }
 
-               /*
-                       u16 command
-                       u16 number of files requested
-                       for each file {
-                               u16 length of name
-                               string name
+               m_media_count = file_requests.size();
+               m_media_receive_started = true;
+
+               if (remote_media == "" || !USE_CURL) {
+                       request_media(file_requests);
+               } else {
+                       #if USE_CURL
+                       for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i) {
+                               m_media_fetch_threads.push_back(MediaFetchThread(this));
                        }
-               */
-               std::ostringstream os(std::ios_base::binary);
-               writeU16(os, TOSERVER_REQUEST_MEDIA);
-               writeU16(os, file_requests.size());
 
-               for(core::list<MediaRequest>::Iterator i = file_requests.begin();
-                               i != file_requests.end(); i++) {
-                       os<<serializeString(i->name);
-               }
+                       core::list<MediaFetchThread>::Iterator cur = m_media_fetch_threads.begin();
+                       for(core::list<MediaRequest>::Iterator i = file_requests.begin();
+                                       i != file_requests.end(); i++) {
+                               cur->m_file_requests.push_back(*i);
+                               cur++;
+                               if (cur == m_media_fetch_threads.end())
+                                       cur = m_media_fetch_threads.begin();
+                       }
+                       for (core::list<MediaFetchThread>::Iterator i = m_media_fetch_threads.begin();
+                                       i != m_media_fetch_threads.end(); i++) {
+                               i->m_remote_url = remote_media;
+                               i->Start();
+                       }
+                       #endif
 
-               // Make data buffer
-               std::string s = os.str();
-               SharedBuffer<u8> data((u8*)s.c_str(), s.size());
-               // Send as reliable
-               Send(0, data, true);
-               infostream<<"Client: Sending media request list to server ("
-                               <<file_requests.size()<<" files)"<<std::endl;
+                       // notify server we received everything
+                       std::ostringstream os(std::ios_base::binary);
+                       writeU16(os, TOSERVER_RECEIVED_MEDIA);
+                       std::string s = os.str();
+                       SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+                       // Send as reliable
+                       Send(0, data, true);
+               }
+               ClientEvent event;
+               event.type = CE_TEXTURES_UPDATED;
+               m_client_event_queue.push_back(event);
        }
        else if(command == TOCLIENT_MEDIA)
        {
+               if (m_media_count == 0)
+                       return;
                std::string datastring((char*)&data[2], datasize-2);
                std::istringstream is(datastring, std::ios_base::binary);
 
@@ -1566,17 +1706,12 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
                */
                int num_bunches = readU16(is);
                int bunch_i = readU16(is);
-               if(num_bunches >= 2)
-                       m_media_receive_progress = (float)bunch_i / (float)(num_bunches - 1);
-               else
-                       m_media_receive_progress = 1.0;
-               if(bunch_i == num_bunches - 1)
-                       m_media_received = true;
                int num_files = readU32(is);
                infostream<<"Client: Received files: bunch "<<bunch_i<<"/"
                                <<num_bunches<<" files="<<num_files
                                <<" size="<<datasize<<std::endl;
                for(int i=0; i<num_files; i++){
+                       m_media_received_count++;
                        std::string name = deSerializeString(is);
                        std::string data = deSerializeLongString(is);
 
@@ -2458,7 +2593,7 @@ void Client::afterContentReceived()
        infostream<<"Client::afterContentReceived() started"<<std::endl;
        assert(m_itemdef_received);
        assert(m_nodedef_received);
-       assert(m_media_received);
+       assert(texturesReceived());
        
        // remove the information about which checksum each texture
        // ought to have
index f85e8ac7be643a5af87b0f7d049780e347675022..c6858f5497c42847951ec2f4666eb4ac05858134 100644 (file)
@@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "filesys.h"
 #include "filecache.h"
 #include "localplayer.h"
+#include "server.h"
 #include "util/pointedthing.h"
 
 struct MeshMakeData;
@@ -129,6 +130,24 @@ public:
        IGameDef *m_gamedef;
 };
 
+class MediaFetchThread : public SimpleThread
+{
+public:
+
+       MediaFetchThread(IGameDef *gamedef):
+               m_gamedef(gamedef)
+       {
+       }
+
+       void * Thread();
+
+       core::list<MediaRequest> m_file_requests;
+       MutexedQueue<std::pair<std::string, std::string> > m_file_data;
+       core::list<MediaRequest> m_failed;
+       std::string m_remote_url;
+       IGameDef *m_gamedef;
+};
+
 enum ClientEventType
 {
        CE_NONE,
@@ -289,10 +308,13 @@ public:
        { return m_access_denied_reason; }
 
        float mediaReceiveProgress()
-       { return m_media_receive_progress; }
+       {
+               if (!m_media_receive_started) return 0;
+               return 1.0 * m_media_received_count / m_media_count;
+       }
 
        bool texturesReceived()
-       { return m_media_received; }
+       { return m_media_receive_started && m_media_received_count == m_media_count; }
        bool itemdefReceived()
        { return m_itemdef_received; }
        bool nodedefReceived()
@@ -318,7 +340,9 @@ private:
        
        // Insert a media file appropriately into the appropriate manager
        bool loadMedia(const std::string &data, const std::string &filename);
-       
+
+       void request_media(const core::list<MediaRequest> &file_requests);
+
        // Virtual methods from con::PeerHandler
        void peerAdded(con::Peer *peer);
        void deletingPeer(con::Peer *peer, bool timeout);
@@ -347,6 +371,7 @@ private:
        MtEventManager *m_event;
 
        MeshUpdateThread m_mesh_update_thread;
+       core::list<MediaFetchThread> m_media_fetch_threads;
        ClientEnvironment m_env;
        con::Connection m_con;
        IrrlichtDevice *m_device;
@@ -375,8 +400,9 @@ private:
        FileCache m_media_cache;
        // Mapping from media file name to SHA1 checksum
        core::map<std::string, std::string> m_media_name_sha1_map;
-       float m_media_receive_progress;
-       bool m_media_received;
+       bool m_media_receive_started;
+       u32 m_media_count;
+       u32 m_media_received_count;
        bool m_itemdef_received;
        bool m_nodedef_received;
        friend class FarMesh;
index 6f9396c02ceaf607689b0eccaf1235eb513e777a..db551a90c410c37164a8de69e80c4a352047c3a2 100644 (file)
@@ -267,6 +267,8 @@ enum ToClientCommand
                        u32 length of data
                        data
                }
+               u16 length of remote media server url (if applicable)
+               string url
        */
        
        TOCLIENT_TOOLDEF = 0x39,
@@ -571,6 +573,10 @@ enum ToServerCommand
                }
         */
 
+       TOSERVER_RECEIVED_MEDIA = 0x41,
+       /*
+               u16 command
+       */
 };
 
 #endif
index c2bdc967040a0c3e72c427dfafd1e26a76bb1a6c..4853d854f73783b48c9d167ae487f84e5399fbd1 100644 (file)
@@ -7,6 +7,7 @@
 #define CMAKE_VERSION_STRING "@VERSION_STRING@"
 #define CMAKE_RUN_IN_PLACE @RUN_IN_PLACE@
 #define CMAKE_USE_GETTEXT @USE_GETTEXT@
+#define CMAKE_USE_CURL @USE_CURL@
 #define CMAKE_USE_SOUND @USE_SOUND@
 #define CMAKE_STATIC_SHAREDIR "@SHAREDIR@"
 
index aedca8b20a61269ba8f6402291eb3a078ef648a1..f37ec0fed466d78a3585930df8ff2ad675a692fa 100644 (file)
@@ -11,6 +11,7 @@
 #define RUN_IN_PLACE 0
 #define USE_GETTEXT 0
 #define USE_SOUND 0
+#define USE_CURL 0
 #define STATIC_SHAREDIR ""
 #define BUILD_INFO "non-cmake"
 
@@ -26,6 +27,8 @@
        #define USE_GETTEXT CMAKE_USE_GETTEXT
        #undef USE_SOUND
        #define USE_SOUND CMAKE_USE_SOUND
+       #undef USE_CURL
+       #define USE_CURL CMAKE_USE_CURL
        #undef STATIC_SHAREDIR
        #define STATIC_SHAREDIR CMAKE_STATIC_SHAREDIR
        #undef BUILD_INFO
index d824d292387542a50975819f13babc54a469b2ce..7cb781276b1da9d29def86ca15e68d03d97615fc 100644 (file)
@@ -119,6 +119,8 @@ void set_default_settings(Settings *settings)
        settings->setDefault("preload_item_visuals", "true");
        settings->setDefault("enable_shaders", "2");
 
+       settings->setDefault("media_fetch_threads", "8");
+
        // Server stuff
        // "map-dir" doesn't exist by default.
        settings->setDefault("default_game", "minetest");
@@ -158,5 +160,6 @@ void set_default_settings(Settings *settings)
        settings->setDefault("congestion_control_aim_rtt", "0.2");
        settings->setDefault("congestion_control_max_rate", "400");
        settings->setDefault("congestion_control_min_rate", "10");
+       settings->setDefault("remote_media", "");
 }
 
index 2449f423659f21418b2664e76d7aa0485bc4303a..4b43a3205dd5583c38ebd354b6363a16eb9aa0d6 100644 (file)
@@ -2885,6 +2885,9 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
                // (definitions and files)
                getClient(peer_id)->definitions_sent = true;
        }
+       else if(command == TOSERVER_RECEIVED_MEDIA) {
+               getClient(peer_id)->definitions_sent = true;
+       }
        else if(command == TOSERVER_INTERACT)
        {
                std::string datastring((char*)&data[2], datasize-2);
@@ -4217,6 +4220,7 @@ void Server::sendMediaAnnouncement(u16 peer_id)
                os<<serializeString(j->name);
                os<<serializeString(j->sha1_digest);
        }
+       os<<serializeString(g_settings->get("remote_media"));
 
        // Make data buffer
        std::string s = os.str();
@@ -4224,7 +4228,6 @@ void Server::sendMediaAnnouncement(u16 peer_id)
 
        // Send as reliable
        m_con.Send(peer_id, 0, data, true);
-
 }
 
 struct SendableMedia
index fb39a24c3a3ad9e0481c22f7cec9210b39651e77..215ac299d13cab211219cda3bfc55c1b772a58ca 100644 (file)
@@ -41,3 +41,9 @@ std::string translatePassword(std::string playername, std::wstring password)
        return pwd;
 }
 
+size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata) {
+    std::ostringstream *stream = (std::ostringstream*)userdata;
+    size_t count = size * nmemb;
+    stream->write(ptr, count);
+    return count;
+}
index 71b11de3d1109cd8687fd6dfa91af21c6d754912..58274c6773cd7a3807c52c7df7674e29734a81fc 100644 (file)
@@ -282,6 +282,7 @@ inline std::string wrap_rows(const std::string &from, u32 rowlen)
 }
 
 std::string translatePassword(std::string playername, std::wstring password);
+size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata);
 
 #endif