UpdateFetcher Class Reference

#include <UpdateFetcher.h>

Classes
struct	AppliedFileEntry
struct	DirectoryEntry
struct	PathCompare

Public Member Functions
	UpdateFetcher (Path const &updateDirectory, std::function< void(std::string const &)> const &apply, std::function< void(Path const &path)> const &applyFile, std::function< QueryResult(std::string const &)> const &retrieve)
UpdateResult	Update (bool const redundancyChecks, bool const allowRehash, bool const archivedRedundancy, int32 const cleanDeadReferencesMaxCount) const

Private Types
enum	UpdateMode { MODE_APPLY, MODE_REHASH }
enum	State { RELEASED, ARCHIVED }
typedef boost::filesystem::path	Path
typedef std::pair< Path, State >	LocaleFileEntry
typedef std::set < LocaleFileEntry, PathCompare >	LocaleFileStorage
typedef std::unordered_map < std::string, std::string >	HashToFileNameStorage
typedef std::unordered_map < std::string, AppliedFileEntry >	AppliedFileStorage
typedef std::vector < UpdateFetcher::DirectoryEntry >	DirectoryStorage

Private Member Functions
LocaleFileStorage	GetFileList () const
void	FillFileListRecursively (Path const &path, LocaleFileStorage &storage, State const state, uint32 const depth) const
DirectoryStorage	ReceiveIncludedDirectories () const
AppliedFileStorage	ReceiveAppliedFiles () const
std::string	ReadSQLUpdate (Path const &file) const
std::string	CalculateHash (std::string const &query) const
uint32	Apply (Path const &path) const
void	UpdateEntry (AppliedFileEntry const &entry, uint32 const speed=0) const
void	RenameEntry (std::string const &from, std::string const &to) const
void	CleanUp (AppliedFileStorage const &storage) const
void	UpdateState (std::string const &name, State const state) const

Private Attributes
Path const	_sourceDirectory
std::function< void(std::string const &)> const	_apply
std::function< void(Path const &path)> const	_applyFile
std::function< QueryResult(std::string const &)> const	_retrieve

Member Typedef Documentation

typedef std::unordered_map<std::string, AppliedFileEntry> UpdateFetcher::AppliedFileStorage

private

typedef std::vector<UpdateFetcher::DirectoryEntry> UpdateFetcher::DirectoryStorage

private

typedef std::unordered_map<std::string, std::string> UpdateFetcher::HashToFileNameStorage

private

typedef std::pair<Path, State> UpdateFetcher::LocaleFileEntry

private

typedef std::set<LocaleFileEntry, PathCompare> UpdateFetcher::LocaleFileStorage

private

typedef boost::filesystem::path UpdateFetcher::Path

private

Member Enumeration Documentation

enum UpdateFetcher::State

private

Enumerator
RELEASED
ARCHIVED

 {
 RELEASED,
 ARCHIVED
 };

enum UpdateFetcher::UpdateMode

private

Enumerator
MODE_APPLY
MODE_REHASH

 {
 MODE_APPLY,
 MODE_REHASH
 };

Constructor & Destructor Documentation

UpdateFetcher::UpdateFetcher	(	Path const &	updateDirectory,
		std::function< void(std::string const &)> const &	apply,
		std::function< void(Path const &path)> const &	applyFile,
		std::function< QueryResult(std::string const &)> const &	retrieve
	)

 :
 _sourceDirectory(sourceDirectory), _apply(apply), _applyFile(applyFile),
 _retrieve(retrieve)
{
}

Member Function Documentation

uint32 UpdateFetcher::Apply ( Path const &  path ) const

private

{
 using Time = std::chrono::high_resolution_clock;

 // Benchmark query speed
 auto const begin = Time::now();

 // Update database
 _applyFile(path);

 // Return time the query took to apply
 return uint32(std::chrono::duration_cast<std::chrono::milliseconds>(Time::now() - begin).count());
}

Here is the caller graph for this function:

std::string UpdateFetcher::CalculateHash ( std::string const &  query ) const

private

{
 // Calculate a Sha1 hash based on query content.
 unsigned char digest[SHA_DIGEST_LENGTH];
 SHA1((unsigned char*)query.c_str(), query.length(), (unsigned char*)&digest);

 return ByteArrayToHexStr(digest, SHA_DIGEST_LENGTH);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void UpdateFetcher::CleanUp ( AppliedFileStorage const &  storage ) const

private

{
 if (storage.empty())
 return;

 std::stringstream update;
 size_t remaining = storage.size();

 update << "DELETE FROM `updates` WHERE `name` IN(";

 for (auto const& entry : storage)
 {
 update << "\"" << entry.first << "\"";
 if ((--remaining) > 0)
 update << ", ";
 }

 update << ")";

 // Update database
 _apply(update.str());
}

Here is the caller graph for this function:

void UpdateFetcher::FillFileListRecursively ( Path const &  path, LocaleFileStorage &  storage, State const  state, uint32 const  depth  ) const

private

{
 static uint32 const MAX_DEPTH = 10;
 static directory_iterator const end;

 for (directory_iterator itr(path); itr != end; ++itr)
 {
 if (is_directory(itr->path()))
 {
 if (depth < MAX_DEPTH)
 FillFileListRecursively(itr->path(), storage, state, depth + 1);
 }
 else if (itr->path().extension() == ".sql")
 {
 TC_LOG_TRACE("sql.updates", "Added locale file \"%s\".", itr->path().filename().generic_string().c_str());

 LocaleFileEntry const entry = { itr->path(), state };

 // Check for doubled filenames
 // Since elements are only compared through their filenames this is ok
 if (storage.find(entry) != storage.end())
 {
 TC_LOG_FATAL("sql.updates", "Duplicated filename occurred \"%s\", since updates are ordered " \
 "through its filename every name needs to be unique!", itr->path().generic_string().c_str());

 throw UpdateException("Updating failed, see the log for details.");
 }

 storage.insert(entry);
 }
 }
}

Here is the caller graph for this function:

UpdateFetcher::LocaleFileStorage UpdateFetcher::GetFileList ( ) const

private

{
 LocaleFileStorage files;
 DirectoryStorage directories = ReceiveIncludedDirectories();
 for (auto const& entry : directories)
 FillFileListRecursively(entry.path, files, entry.state, 1);

 return files;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string UpdateFetcher::ReadSQLUpdate ( Path const &  file ) const

private

{
 std::ifstream in(file.c_str());
 if (!in.is_open())
 {
 TC_LOG_FATAL("sql.updates", "Failed to open the sql update \"%s\" for reading! "
 "Stopping the server to keep the database integrity, "
 "try to identify and solve the issue or disabled the database updater.",
 file.generic_string().c_str());

 throw UpdateException("Opening the sql update failed!");
 }

 auto update = [&in] {
 std::ostringstream ss;
 ss << in.rdbuf();
 return ss.str();
 }();

 in.close();
 return update;
}

Here is the caller graph for this function:

UpdateFetcher::AppliedFileStorage UpdateFetcher::ReceiveAppliedFiles ( ) const

private

{
 AppliedFileStorage map;

 QueryResult result = _retrieve("SELECT `name`, `hash`, `state`, UNIX_TIMESTAMP(`timestamp`) FROM `updates` ORDER BY `name` ASC");
 if (!result)
 return map;

 do
 {
 Field* fields = result->Fetch();

 AppliedFileEntry const entry = { fields[0].GetString(), fields[1].GetString(),
 AppliedFileEntry::StateConvert(fields[2].GetString()), fields[3].GetUInt64() };

 map.insert(std::make_pair(entry.name, entry));
 }
 while (result->NextRow());

 return map;
}

Here is the call graph for this function:

Here is the caller graph for this function:

UpdateFetcher::DirectoryStorage UpdateFetcher::ReceiveIncludedDirectories ( ) const

private

{
 DirectoryStorage directories;

 QueryResult const result = _retrieve("SELECT `path`, `state` FROM `updates_include`");
 if (!result)
 return directories;

 do
 {
 Field* fields = result->Fetch();

 std::string path = fields[0].GetString();
 if (path.substr(0, 1) == "$")
 path = _sourceDirectory.generic_string() + path.substr(1);

 Path const p(path);

 if (!is_directory(p))
 {
 TC_LOG_WARN("sql.updates", "DBUpdater: Given update include directory \"%s\" isn't existing, skipped!", p.generic_string().c_str());
 continue;
 }

 DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert(fields[1].GetString()) };
 directories.push_back(entry);

 TC_LOG_TRACE("sql.updates", "Added applied file \"%s\" from remote.", p.filename().generic_string().c_str());

 } while (result->NextRow());

 return directories;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void UpdateFetcher::RenameEntry ( std::string const &  from, std::string const &  to  ) const

private

{
 // Delete target if it exists
 {
 std::string const update = "DELETE FROM `updates` WHERE `name`=\"" + to + "\"";

 // Update database
 _apply(update);
 }

 // Rename
 {
 std::string const update = "UPDATE `updates` SET `name`=\"" + to + "\" WHERE `name`=\"" + from + "\"";

 // Update database
 _apply(update);
 }
}

Here is the caller graph for this function:

UpdateResult UpdateFetcher::Update	(	bool const	redundancyChecks,
		bool const	allowRehash,
		bool const	archivedRedundancy,
		int32 const	cleanDeadReferencesMaxCount
	)		const

{
 LocaleFileStorage const available = GetFileList();
 AppliedFileStorage applied = ReceiveAppliedFiles();

 size_t countRecentUpdates = 0;
 size_t countArchivedUpdates = 0;

 // Count updates
 for (auto const& entry : applied)
 if (entry.second.state == RELEASED)
 ++countRecentUpdates;
 else
 ++countArchivedUpdates;

 // Fill hash to name cache
 HashToFileNameStorage hashToName;
 for (auto entry : applied)
 hashToName.insert(std::make_pair(entry.second.hash, entry.first));

 size_t importedUpdates = 0;

 for (auto const& availableQuery : available)
 {
 TC_LOG_DEBUG("sql.updates", "Checking update \"%s\"...", availableQuery.first.filename().generic_string().c_str());

 AppliedFileStorage::const_iterator iter = applied.find(availableQuery.first.filename().string());
 if (iter != applied.end())
 {
 // If redundancy is disabled skip it since the update is already applied.
 if (!redundancyChecks)
 {
 TC_LOG_DEBUG("sql.updates", ">> Update is already applied, skipping redundancy checks.");
 applied.erase(iter);
 continue;
 }

 // If the update is in an archived directory and is marked as archived in our database skip redundancy checks (archived updates never change).
 if (!archivedRedundancy && (iter->second.state == ARCHIVED) && (availableQuery.second == ARCHIVED))
 {
 TC_LOG_DEBUG("sql.updates", ">> Update is archived and marked as archived in database, skipping redundancy checks.");
 applied.erase(iter);
 continue;
 }
 }

 // Calculate hash
 std::string const hash =
 CalculateHash(ReadSQLUpdate(availableQuery.first));

 UpdateMode mode = MODE_APPLY;

 // Update is not in our applied list
 if (iter == applied.end())
 {
 // Catch renames (different filename but same hash)
 HashToFileNameStorage::const_iterator const hashIter = hashToName.find(hash);
 if (hashIter != hashToName.end())
 {
 // Check if the original file was removed if not we've got a problem.
 LocaleFileStorage::const_iterator localeIter;
 // Push localeIter forward
 for (localeIter = available.begin(); (localeIter != available.end()) &&
 (localeIter->first.filename().string() != hashIter->second); ++localeIter);

 // Conflict!
 if (localeIter != available.end())
 {
 TC_LOG_WARN("sql.updates", ">> Seems like update \"%s\" \'%s\' was renamed, but the old file is still there! " \
 "Trade it as a new file! (Probably its an unmodified copy of file \"%s\")",
 availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str(),
 localeIter->first.filename().string().c_str());
 }
 // Its save to trade the file as renamed here
 else
 {
 TC_LOG_INFO("sql.updates", ">> Renaming update \"%s\" to \"%s\" \'%s\'.",
 hashIter->second.c_str(), availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());

 RenameEntry(hashIter->second, availableQuery.first.filename().string());
 applied.erase(hashIter->second);
 continue;
 }
 }
 // Apply the update if it was never seen before.
 else
 {
 TC_LOG_INFO("sql.updates", ">> Applying update \"%s\" \'%s\'...",
 availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
 }
 }
 // Rehash the update entry if it is contained in our database but with an empty hash.
 else if (allowRehash && iter->second.hash.empty())
 {
 mode = MODE_REHASH;

 TC_LOG_INFO("sql.updates", ">> Re-hashing update \"%s\" \'%s\'...", availableQuery.first.filename().string().c_str(),
 hash.substr(0, 7).c_str());
 }
 else
 {
 // If the hash of the files differs from the one stored in our database reapply the update (because it was changed).
 if (iter->second.hash != hash)
 {
 TC_LOG_INFO("sql.updates", ">> Reapplying update \"%s\" \'%s\' -> \'%s\' (it changed)...", availableQuery.first.filename().string().c_str(),
 iter->second.hash.substr(0, 7).c_str(), hash.substr(0, 7).c_str());
 }
 else
 {
 // If the file wasn't changed and just moved update its state if necessary.
 if (iter->second.state != availableQuery.second)
 {
 TC_LOG_DEBUG("sql.updates", ">> Updating state of \"%s\" to \'%s\'...",
 availableQuery.first.filename().string().c_str(), AppliedFileEntry::StateConvert(availableQuery.second).c_str());

 UpdateState(availableQuery.first.filename().string(), availableQuery.second);
 }

 TC_LOG_DEBUG("sql.updates", ">> Update is already applied and is matching hash \'%s\'.", hash.substr(0, 7).c_str());

 applied.erase(iter);
 continue;
 }
 }

 uint32 speed = 0;
 AppliedFileEntry const file = { availableQuery.first.filename().string(), hash, availableQuery.second, 0 };

 switch (mode)
 {
 case MODE_APPLY:
 speed = Apply(availableQuery.first);
 /*no break*/
 case MODE_REHASH:
 UpdateEntry(file, speed);
 break;
 }

 if (iter != applied.end())
 applied.erase(iter);

 if (mode == MODE_APPLY)
 ++importedUpdates;
 }

 // Cleanup up orphaned entries if enabled
 if (!applied.empty())
 {
 bool const doCleanup = (cleanDeadReferencesMaxCount < 0) || (applied.size() <= static_cast<size_t>(cleanDeadReferencesMaxCount));

 for (auto const& entry : applied)
 {
 TC_LOG_WARN("sql.updates", ">> File \'%s\' was applied to the database but is missing in" \
 " your update directory now!", entry.first.c_str());

 if (doCleanup)
 TC_LOG_INFO("sql.updates", "Deleting orphaned entry \'%s\'...", entry.first.c_str());
 }

 if (doCleanup)
 CleanUp(applied);
 else
 {
 TC_LOG_ERROR("sql.updates", "Cleanup is disabled! There are " SZFMTD " dirty files that were applied to your database " \
 "but are now missing in your source directory!", applied.size());
 }
 }

 return UpdateResult(importedUpdates, countRecentUpdates, countArchivedUpdates);
}

Here is the call graph for this function:

void UpdateFetcher::UpdateEntry ( AppliedFileEntry const &  entry, uint32 const  speed = 0  ) const

private

{
 std::string const update = "REPLACE INTO `updates` (`name`, `hash`, `state`, `speed`) VALUES (\"" +
 entry.name + "\", \"" + entry.hash + "\", \'" + entry.GetStateAsString() + "\', " + std::to_string(speed) + ")";

 // Update database
 _apply(update);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void UpdateFetcher::UpdateState ( std::string const &  name, State const  state  ) const

private

{
 std::string const update = "UPDATE `updates` SET `state`=\'" + AppliedFileEntry::StateConvert(state) + "\' WHERE `name`=\"" + name + "\"";

 // Update database
 _apply(update);
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

std::function<void(std::string const&)> const UpdateFetcher::_apply

private

std::function<void(Path const& path)> const UpdateFetcher::_applyFile

private

std::function<QueryResult(std::string const&)> const UpdateFetcher::_retrieve

private

Path const UpdateFetcher::_sourceDirectory

private

---

The documentation for this class was generated from the following files:

• src/server/database/Updater/UpdateFetcher.h
• src/server/database/Updater/UpdateFetcher.cpp