From e09fec3373abcf836635dc33a95e9b9975058eb9 Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Wed, 18 May 2011 16:48:27 +0100 Subject: [PATCH] New map directory structure that avoids map size being limited by filesystem --- src/filesys.cpp | 19 ++++++++ src/filesys.h | 3 ++ src/map.cpp | 124 +++++++++++++++++++++++++++++++++--------------- src/map.h | 12 ++--- 4 files changed, 111 insertions(+), 47 deletions(-) diff --git a/src/filesys.cpp b/src/filesys.cpp index 287090e8..8248a13d 100644 --- a/src/filesys.cpp +++ b/src/filesys.cpp @@ -290,5 +290,24 @@ bool RecursiveDeleteContent(std::string path) return true; } +bool CreateAllDirs(std::string path) +{ + + size_t pos; + std::vector tocreate; + std::string basepath = path; + while(!PathExists(basepath)) + { + tocreate.push_back(basepath); + pos = basepath.rfind('/'); + if(pos == std::string::npos) + return false; + basepath = basepath.substr(0,pos); + } + for(int i=tocreate.size()-1;i>=0;i--) + CreateDir(tocreate[i]); + return true; +} + } // namespace fs diff --git a/src/filesys.h b/src/filesys.h index 4dd90b84..b74b34f3 100644 --- a/src/filesys.h +++ b/src/filesys.h @@ -38,6 +38,9 @@ std::vector GetDirListing(std::string path); // Returns true if already exists bool CreateDir(std::string path); +// Create all directories on the given path that don't already exist. +bool CreateAllDirs(std::string path); + bool PathExists(std::string path); // Only pass full paths to this one. True on success. diff --git a/src/map.cpp b/src/map.cpp index 63f01dde..cd2ba915 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -4875,9 +4875,9 @@ plan_b: return (s16)level; } -void ServerMap::createDir(std::string path) +void ServerMap::createDirs(std::string path) { - if(fs::CreateDir(path) == false) + if(fs::CreateAllDirs(path) == false) { m_dout<getPos(); - createDir(m_savedir); - createDir(m_savedir+"/sectors"); std::string dir = getSectorDir(pos); - createDir(dir); + createDirs(dir); std::string fullpath = dir + "/meta"; std::ofstream o(fullpath.c_str(), std::ios_base::binary); @@ -5282,22 +5303,21 @@ void ServerMap::saveSectorMeta(ServerMapSector *sector) sector->differs_from_disk = false; } -MapSector* ServerMap::loadSectorMeta(std::string dirname) +MapSector* ServerMap::loadSectorMeta(std::string sectordir, bool save_after_load) { DSTACK(__FUNCTION_NAME); // Get destination - v2s16 p2d = getSectorPos(dirname); - std::string dir = m_savedir + "/sectors/" + dirname; + v2s16 p2d = getSectorPos(sectordir); ServerMapSector *sector = NULL; - - std::string fullpath = dir + "/meta"; + + std::string fullpath = sectordir + "/meta"; std::ifstream is(fullpath.c_str(), std::ios_base::binary); if(is.good() == false) { // If the directory exists anyway, it probably is in some old // format. Just go ahead and create the sector. - if(fs::PathExists(dir)) + if(fs::PathExists(sectordir)) { dstream<<"ServerMap::loadSectorMeta(): Sector metafile " <differs_from_disk = false; @@ -5323,14 +5347,31 @@ MapSector* ServerMap::loadSectorMeta(std::string dirname) bool ServerMap::loadSectorFull(v2s16 p2d) { DSTACK(__FUNCTION_NAME); - std::string sectorsubdir = getSectorSubDir(p2d); MapSector *sector = NULL; + // The directory layout we're going to load from. + // 1 - original sectors/xxxxzzzz/ + // 2 - new sectors2/xxx/zzz/ + // If we load from anything but the latest structure, we will + // immediately save to the new one, and remove the old. + int loadlayout = 1; + std::string sectordir1 = getSectorDir(p2d, 1); + std::string sectordir; + if(fs::PathExists(sectordir1)) + { + sectordir = sectordir1; + } + else + { + loadlayout = 2; + sectordir = getSectorDir(p2d, 2); + } + //JMutexAutoLock lock(m_sector_mutex); // Bulk comment-out try{ - sector = loadSectorMeta(sectorsubdir); + sector = loadSectorMeta(sectordir, loadlayout != 2); } catch(InvalidFilenameException &e) { @@ -5349,7 +5390,7 @@ bool ServerMap::loadSectorFull(v2s16 p2d) Load blocks */ std::vector list2 = fs::GetDirListing - (m_savedir+"/sectors/"+sectorsubdir); + (sectordir); std::vector::iterator i2; for(i2=list2.begin(); i2!=list2.end(); i2++) { @@ -5357,16 +5398,25 @@ bool ServerMap::loadSectorFull(v2s16 p2d) if(i2->dir) continue; try{ - loadBlock(sectorsubdir, i2->name, sector); + loadBlock(sectordir, i2->name, sector, loadlayout != 2); } catch(InvalidFilenameException &e) { // This catches unknown crap in directory } } + + if(loadlayout != 2) + { + dstream<<"Sector converted to new layout - deleting "<< + sectordir1<getPos(); v2s16 p2d(p3d.X, p3d.Z); - createDir(m_savedir); - createDir(m_savedir+"/sectors"); std::string dir = getSectorDir(p2d); - createDir(dir); + createDirs(dir); - // Block file is map/sectors/xxxxxxxx/xxxx char cc[5]; snprintf(cc, 5, "%.4x", (unsigned int)p3d.Y&0xffff); std::string fullpath = dir + "/" + cc; @@ -5427,12 +5474,11 @@ void ServerMap::saveBlock(MapBlock *block) block->resetChangedFlag(); } -void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSector *sector) +void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load) { DSTACK(__FUNCTION_NAME); - // Block file is map/sectors/xxxxxxxx/xxxx - std::string fullpath = m_savedir+"/sectors/"+sectordir+"/"+blockfile; + std::string fullpath = sectordir+"/"+blockfile; try{ std::ifstream is(fullpath.c_str(), std::ios_base::binary); @@ -5496,7 +5542,7 @@ void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSecto */ // Save old format blocks in new format - if(version < SER_FMT_VER_HIGHEST) + if(version < SER_FMT_VER_HIGHEST || save_after_load) { saveBlock(block); } diff --git a/src/map.h b/src/map.h index fa52dc26..09154547 100644 --- a/src/map.h +++ b/src/map.h @@ -545,13 +545,9 @@ public: Misc. helper functions for fiddling with directory and file names when saving */ - void createDir(std::string path); - void createSaveDir(); - // returns something like "xxxxxxxx" - std::string getSectorSubDir(v2s16 pos); + void createDirs(std::string path); // returns something like "map/sectors/xxxxxxxx" - std::string getSectorDir(v2s16 pos); - std::string createSectorDir(v2s16 pos); + std::string getSectorDir(v2s16 pos, int layout = 2); // dirname: final directory name v2s16 getSectorPos(std::string dirname); v3s16 getBlockPos(std::string sectordir, std::string blockfile); @@ -572,7 +568,7 @@ public: // (no MapBlocks) // DEPRECATED? Sectors have no metadata anymore. void saveSectorMeta(ServerMapSector *sector); - MapSector* loadSectorMeta(std::string dirname); + MapSector* loadSectorMeta(std::string dirname, bool save_after_load); // Full load of a sector including all blocks. // returns true on success, false on failure. @@ -583,7 +579,7 @@ public: void saveBlock(MapBlock *block); // This will generate a sector with getSector if not found. - void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector); + void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load=false); // For debug printing virtual void PrintInfo(std::ostream &out); -- 2.30.2