Aegisub/aegisub/libaegisub/common/charset_ucd.cpp

// Copyright (c) 2010, Amar Takhar <verm@aegisub.org>
//
// Permission to use, copy, modify, and distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

/// @file charset_ucd.cpp
/// @brief Character set detection using Universalchardet
/// @ingroup libaegisub

#include "charset_ucd.h"

#include "libaegisub/io.h"
#include "libaegisub/scoped_ptr.h"

#include "../../universalchardet/nsCharSetProber.h"

namespace agi {
	namespace charset {

UCDetect::UCDetect(const std::string &file)
: nsUniversalDetector(NS_FILTER_ALL)
{
	{
		agi::scoped_ptr<std::ifstream> fp(io::Open(file, true));

		// If it's over 100 MB it's either binary or big enough that we won't
		// be able to do anything useful with it anyway
		fp->seekg(0, std::ios::end);
		if (fp->tellg() > 100 * 1024 * 1024) {
			list.emplace_back(1.f, "binary");
			return;
		}
		fp->seekg(0, std::ios::beg);

		std::streamsize binaryish = 0;
		std::streamsize bytes = 0;

		while (!mDone && *fp) {
			char buf[4096];
			fp->read(buf, sizeof(buf));
			std::streamsize read = fp->gcount();
			HandleData(buf, (PRUint32)read);

			// A dumb heuristic to detect binary files
			if (!mDone) {
				bytes += read;
				for (std::streamsize i = 0; i < read; ++i) {
					if ((unsigned char)buf[i] < 32 && (buf[i] != '\r' && buf[i] != '\n' && buf[i] != '\t'))
						++binaryish;
				}

				if (binaryish > bytes / 8) {
					list.emplace_back(1.f, "binary");
					return;
				}
			}
		}
	}

	DataEnd();

	if (mDetectedCharset)
		list.emplace_back(1.f, mDetectedCharset);
	else {
		switch (mInputState) {
			case eHighbyte: {
				for (PRInt32 i=0; i<NUM_OF_CHARSET_PROBERS; i++) {
					if (!mCharSetProbers[i]) continue;

					float conf = mCharSetProbers[i]->GetConfidence();
					if (conf > 0.01f)
						list.emplace_back(conf, mCharSetProbers[i]->GetCharSetName());
				}

				break;
			}
			case ePureAscii:
				list.emplace_back(1.f, "US-ASCII");
				break;

			default:
				throw UnknownCharset("Unknown character set.");
		}

		if (list.empty() && (mInputState == eHighbyte))
			throw UnknownCharset("Unknown character set.");

		typedef std::pair<float, std::string> const& result;
		sort(begin(list), end(list), [](result lft, result rgt) { return lft.first > rgt.first; });
	}
}

std::string UCDetect::Single() const {
	return list.front().second;
}

	} // namespace util
} // namespace agi